1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Thanks to
10 * Niall Sweeny <niall.sweeny@fonjax.com> for MS SQL support
11 * Marcel Leuthi <mluethi@mlsystems.ch> for Oracle support
12 * Denes
13 * Chris Clark
14 * clach05
15 * Denes Lengyel
16 * and many others who have contributed to current and previous versions
17
18 This file contains the DAL support for many relational databases,
19 including:
20 - SQLite & SpatiaLite
21 - MySQL
22 - Postgres
23 - Firebird
24 - Oracle
25 - MS SQL
26 - DB2
27 - Interbase
28 - Ingres
29 - Informix (9+ and SE)
30 - SapDB (experimental)
31 - Cubrid (experimental)
32 - CouchDB (experimental)
33 - MongoDB (in progress)
34 - Google:nosql
35 - Google:sql
36 - Teradata
37 - IMAP (experimental)
38
39 Example of usage:
40
41 >>> # from dal import DAL, Field
42
43 ### create DAL connection (and create DB if it doesn't exist)
44 >>> db = DAL(('sqlite://storage.sqlite','mysql://a:b@localhost/x'),
45 ... folder=None)
46
47 ### define a table 'person' (create/alter as necessary)
48 >>> person = db.define_table('person',Field('name','string'))
49
50 ### insert a record
51 >>> id = person.insert(name='James')
52
53 ### retrieve it by id
54 >>> james = person(id)
55
56 ### retrieve it by name
57 >>> james = person(name='James')
58
59 ### retrieve it by arbitrary query
60 >>> query = (person.name=='James') & (person.name.startswith('J'))
61 >>> james = db(query).select(person.ALL)[0]
62
63 ### update one record
64 >>> james.update_record(name='Jim')
65 <Row {'id': 1, 'name': 'Jim'}>
66
67 ### update multiple records by query
68 >>> db(person.name.like('J%')).update(name='James')
69 1
70
71 ### delete records by query
72 >>> db(person.name.lower() == 'jim').delete()
73 0
74
75 ### retrieve multiple records (rows)
76 >>> people = db(person).select(orderby=person.name,
77 ... groupby=person.name, limitby=(0,100))
78
79 ### further filter them
80 >>> james = people.find(lambda row: row.name == 'James').first()
81 >>> print james.id, james.name
82 1 James
83
84 ### check aggregates
85 >>> counter = person.id.count()
86 >>> print db(person).select(counter).first()(counter)
87 1
88
89 ### delete one record
90 >>> james.delete_record()
91 1
92
93 ### delete (drop) entire database table
94 >>> person.drop()
95
96 Supported field types:
97 id string text boolean integer double decimal password upload
98 blob time date datetime
99
100 Supported DAL URI strings:
101 'sqlite://test.db'
102 'spatialite://test.db'
103 'sqlite:memory'
104 'spatialite:memory'
105 'jdbc:sqlite://test.db'
106 'mysql://root:none@localhost/test'
107 'postgres://mdipierro:password@localhost/test'
108 'postgres:psycopg2://mdipierro:password@localhost/test'
109 'postgres:pg8000://mdipierro:password@localhost/test'
110 'jdbc:postgres://mdipierro:none@localhost/test'
111 'mssql://web2py:none@A64X2/web2py_test'
112 'mssql2://web2py:none@A64X2/web2py_test' # alternate mappings
113 'oracle://username:password@database'
114 'firebird://user:password@server:3050/database'
115 'db2://DSN=dsn;UID=user;PWD=pass'
116 'firebird://username:password@hostname/database'
117 'firebird_embedded://username:password@c://path'
118 'informix://user:password@server:3050/database'
119 'informixu://user:password@server:3050/database' # unicode informix
120 'google:datastore' # for google app engine datastore
121 'google:sql' # for google app engine with sql (mysql compatible)
122 'teradata://DSN=dsn;UID=user;PWD=pass; DATABASE=database' # experimental
123 'imap://user:password@server:port' # experimental
124
125 For more info:
126 help(DAL)
127 help(Field)
128 """
129
130
131
132
133
134 __all__ = ['DAL', 'Field']
135
136 MAXCHARLENGTH = 2**15
137 DEFAULTLENGTH = {'string':512,
138 'password':512,
139 'upload':512,
140 'text':2**15,
141 'blob':2**31}
142 TIMINGSSIZE = 100
143 SPATIALLIBS = {
144 'Windows':'libspatialite',
145 'Linux':'libspatialite.so',
146 'Darwin':'libspatialite.dylib'
147 }
148 DEFAULT_URI = 'sqlite://dummy.db'
149
150 import re
151 import sys
152 import locale
153 import os
154 import types
155 import datetime
156 import threading
157 import time
158 import csv
159 import cgi
160 import copy
161 import socket
162 import logging
163 import base64
164 import shutil
165 import marshal
166 import decimal
167 import struct
168 import urllib
169 import hashlib
170 import uuid
171 import glob
172 import traceback
173 import platform
174
175 PYTHON_VERSION = sys.version_info[0]
176 if PYTHON_VERSION == 2:
177 import cPickle as pickle
178 import cStringIO as StringIO
179 import copy_reg as copyreg
180 hashlib_md5 = hashlib.md5
181 bytes, unicode = str, unicode
182 else:
183 import pickle
184 from io import StringIO as StringIO
185 import copyreg
186 long = int
187 hashlib_md5 = lambda s: hashlib.md5(bytes(s,'utf8'))
188 bytes, unicode = bytes, str
189
190 CALLABLETYPES = (types.LambdaType, types.FunctionType,
191 types.BuiltinFunctionType,
192 types.MethodType, types.BuiltinMethodType)
193
194 TABLE_ARGS = set(
195 ('migrate','primarykey','fake_migrate','format','redefine',
196 'singular','plural','trigger_name','sequence_name',
197 'common_filter','polymodel','table_class','on_define',))
198
199 SELECT_ARGS = set(
200 ('orderby', 'groupby', 'limitby','required', 'cache', 'left',
201 'distinct', 'having', 'join','for_update', 'processor','cacheable'))
202
203 ogetattr = object.__getattribute__
204 osetattr = object.__setattr__
205 exists = os.path.exists
206 pjoin = os.path.join
207
208
209
210
211 try:
212 from utils import web2py_uuid
213 except (ImportError, SystemError):
214 import uuid
216
217 try:
218 import portalocker
219 have_portalocker = True
220 except ImportError:
221 have_portalocker = False
222
223 try:
224 import serializers
225 have_serializers = True
226 except ImportError:
227 have_serializers = False
228 try:
229 import json as simplejson
230 except ImportError:
231 try:
232 import gluon.contrib.simplejson as simplejson
233 except ImportError:
234 simplejson = None
235
236 try:
237 import validators
238 have_validators = True
239 except (ImportError, SyntaxError):
240 have_validators = False
241
242 LOGGER = logging.getLogger("web2py.dal")
243 DEFAULT = lambda:0
244
245 GLOBAL_LOCKER = threading.RLock()
246 THREAD_LOCAL = threading.local()
247
248
249
250
251 REGEX_TYPE = re.compile('^([\w\_\:]+)')
252 REGEX_DBNAME = re.compile('^(\w+)(\:\w+)*')
253 REGEX_W = re.compile('^\w+$')
254 REGEX_TABLE_DOT_FIELD = re.compile('^(\w+)\.(\w+)$')
255 REGEX_UPLOAD_PATTERN = re.compile('(?P<table>[\w\-]+)\.(?P<field>[\w\-]+)\.(?P<uuidkey>[\w\-]+)\.(?P<name>\w+)\.\w+$')
256 REGEX_CLEANUP_FN = re.compile('[\'"\s;]+')
257 REGEX_UNPACK = re.compile('(?<!\|)\|(?!\|)')
258 REGEX_PYTHON_KEYWORDS = re.compile('^(and|del|from|not|while|as|elif|global|or|with|assert|else|if|pass|yield|break|except|import|print|class|exec|in|raise|continue|finally|is|return|def|for|lambda|try)$')
259 REGEX_SELECT_AS_PARSER = re.compile("\s+AS\s+(\S+)")
260 REGEX_CONST_STRING = re.compile('(\"[^\"]*?\")|(\'[^\']*?\')')
261 REGEX_SEARCH_PATTERN = re.compile('^{[^\.]+\.[^\.]+(\.(lt|gt|le|ge|eq|ne|contains|startswith|year|month|day|hour|minute|second))?(\.not)?}$')
262 REGEX_SQUARE_BRACKETS = re.compile('^.+\[.+\]$')
263 REGEX_STORE_PATTERN = re.compile('\.(?P<e>\w{1,5})$')
264 REGEX_QUOTES = re.compile("'[^']*'")
265 REGEX_ALPHANUMERIC = re.compile('^[0-9a-zA-Z]\w*$')
266 REGEX_PASSWORD = re.compile('\://([^:@]*)\:')
267 REGEX_NOPASSWD = re.compile('(?<=\:)([^:@/]+)(?=@.+)')
268
269
270
271 DRIVERS = []
272
273 try:
274 from new import classobj
275 from google.appengine.ext import db as gae
276 from google.appengine.api import namespace_manager, rdbms
277 from google.appengine.api.datastore_types import Key
278 from google.appengine.ext.db.polymodel import PolyModel
279 DRIVERS.append('google')
280 except ImportError:
281 pass
282
283 if not 'google' in DRIVERS:
284
285 try:
286 from pysqlite2 import dbapi2 as sqlite2
287 DRIVERS.append('SQLite(sqlite2)')
288 except ImportError:
289 LOGGER.debug('no SQLite drivers pysqlite2.dbapi2')
290
291 try:
292 from sqlite3 import dbapi2 as sqlite3
293 DRIVERS.append('SQLite(sqlite3)')
294 except ImportError:
295 LOGGER.debug('no SQLite drivers sqlite3')
296
297 try:
298
299 try:
300 import contrib.pymysql as pymysql
301
302
303 pymysql.ESCAPE_REGEX = re.compile("'")
304 pymysql.ESCAPE_MAP = {"'": "''"}
305
306 except ImportError:
307 import pymysql
308 DRIVERS.append('MySQL(pymysql)')
309 except ImportError:
310 LOGGER.debug('no MySQL driver pymysql')
311
312 try:
313 import MySQLdb
314 DRIVERS.append('MySQL(MySQLdb)')
315 except ImportError:
316 LOGGER.debug('no MySQL driver MySQLDB')
317
318
319 try:
320 import psycopg2
321 from psycopg2.extensions import adapt as psycopg2_adapt
322 DRIVERS.append('PostgreSQL(psycopg2)')
323 except ImportError:
324 LOGGER.debug('no PostgreSQL driver psycopg2')
325
326 try:
327
328 try:
329 import contrib.pg8000.dbapi as pg8000
330 except ImportError:
331 import pg8000.dbapi as pg8000
332 DRIVERS.append('PostgreSQL(pg8000)')
333 except ImportError:
334 LOGGER.debug('no PostgreSQL driver pg8000')
335
336 try:
337 import cx_Oracle
338 DRIVERS.append('Oracle(cx_Oracle)')
339 except ImportError:
340 LOGGER.debug('no Oracle driver cx_Oracle')
341
342 try:
343 try:
344 import pyodbc
345 except ImportError:
346 try:
347 import contrib.pypyodbc as pyodbc
348 except Exception, e:
349 raise ImportError(str(e))
350 DRIVERS.append('MSSQL(pyodbc)')
351 DRIVERS.append('DB2(pyodbc)')
352 DRIVERS.append('Teradata(pyodbc)')
353 except ImportError:
354 LOGGER.debug('no MSSQL/DB2/Teradata driver pyodbc')
355
356 try:
357 import Sybase
358 DRIVERS.append('Sybase(Sybase)')
359 except ImportError:
360 LOGGER.debug('no Sybase driver')
361
362 try:
363 import kinterbasdb
364 DRIVERS.append('Interbase(kinterbasdb)')
365 DRIVERS.append('Firebird(kinterbasdb)')
366 except ImportError:
367 LOGGER.debug('no Firebird/Interbase driver kinterbasdb')
368
369 try:
370 import fdb
371 DRIVERS.append('Firbird(fdb)')
372 except ImportError:
373 LOGGER.debug('no Firebird driver fdb')
374
375 try:
376 import firebirdsql
377 DRIVERS.append('Firebird(firebirdsql)')
378 except ImportError:
379 LOGGER.debug('no Firebird driver firebirdsql')
380
381 try:
382 import informixdb
383 DRIVERS.append('Informix(informixdb)')
384 LOGGER.warning('Informix support is experimental')
385 except ImportError:
386 LOGGER.debug('no Informix driver informixdb')
387
388 try:
389 import sapdb
390 DRIVERS.append('SQL(sapdb)')
391 LOGGER.warning('SAPDB support is experimental')
392 except ImportError:
393 LOGGER.debug('no SAP driver sapdb')
394
395 try:
396 import cubriddb
397 DRIVERS.append('Cubrid(cubriddb)')
398 LOGGER.warning('Cubrid support is experimental')
399 except ImportError:
400 LOGGER.debug('no Cubrid driver cubriddb')
401
402 try:
403 from com.ziclix.python.sql import zxJDBC
404 import java.sql
405
406 from org.sqlite import JDBC
407 zxJDBC_sqlite = java.sql.DriverManager
408 DRIVERS.append('PostgreSQL(zxJDBC)')
409 DRIVERS.append('SQLite(zxJDBC)')
410 LOGGER.warning('zxJDBC support is experimental')
411 is_jdbc = True
412 except ImportError:
413 LOGGER.debug('no SQLite/PostgreSQL driver zxJDBC')
414 is_jdbc = False
415
416 try:
417 import ingresdbi
418 DRIVERS.append('Ingres(ingresdbi)')
419 except ImportError:
420 LOGGER.debug('no Ingres driver ingresdbi')
421
422
423 try:
424 import couchdb
425 DRIVERS.append('CouchDB(couchdb)')
426 except ImportError:
427 LOGGER.debug('no Couchdb driver couchdb')
428
429 try:
430 import pymongo
431 DRIVERS.append('MongoDB(pymongo)')
432 except:
433 LOGGER.debug('no MongoDB driver pymongo')
434
435 try:
436 import imaplib
437 DRIVERS.append('IMAP(imaplib)')
438 except:
439 LOGGER.debug('no IMAP driver imaplib')
440
441 PLURALIZE_RULES = [
442 (re.compile('child$'), re.compile('child$'), 'children'),
443 (re.compile('oot$'), re.compile('oot$'), 'eet'),
444 (re.compile('ooth$'), re.compile('ooth$'), 'eeth'),
445 (re.compile('l[eo]af$'), re.compile('l([eo])af$'), 'l\\1aves'),
446 (re.compile('sis$'), re.compile('sis$'), 'ses'),
447 (re.compile('man$'), re.compile('man$'), 'men'),
448 (re.compile('ife$'), re.compile('ife$'), 'ives'),
449 (re.compile('eau$'), re.compile('eau$'), 'eaux'),
450 (re.compile('lf$'), re.compile('lf$'), 'lves'),
451 (re.compile('[sxz]$'), re.compile('$'), 'es'),
452 (re.compile('[^aeioudgkprt]h$'), re.compile('$'), 'es'),
453 (re.compile('(qu|[^aeiou])y$'), re.compile('y$'), 'ies'),
454 (re.compile('$'), re.compile('$'), 's'),
455 ]
462
465
468
471
473
476
477 if 'google' in DRIVERS:
478
479 is_jdbc = False
482 """
483 GAE decimal implementation
484 """
485 data_type = decimal.Decimal
486
487 - def __init__(self, precision, scale, **kwargs):
488 super(GAEDecimalProperty, self).__init__(self, **kwargs)
489 d = '1.'
490 for x in range(scale):
491 d += '0'
492 self.round = decimal.Decimal(d)
493
501
503 if value is None or value == '':
504 return None
505 else:
506 return decimal.Decimal(value).quantize(self.round)
507
509 value = super(GAEDecimalProperty, self).validate(value)
510 if value is None or isinstance(value, decimal.Decimal):
511 return value
512 elif isinstance(value, basestring):
513 return decimal.Decimal(value)
514 raise gae.BadValueError("Property %s must be a Decimal or string."\
515 % self.name)
516
522
523 POOLS = {}
524 check_active_connection = True
525
526 @staticmethod
529
530
531
532 - def close(self,action='commit',really=True):
549
550 @staticmethod
552 """ to close cleanly databases in a multithreaded environment """
553 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
554 for db_uid, db_group in dbs:
555 for db in db_group:
556 if hasattr(db,'_adapter'):
557 db._adapter.close(action)
558 getattr(THREAD_LOCAL,'db_instances',{}).clear()
559 getattr(THREAD_LOCAL,'db_instances_zombie',{}).clear()
560 if callable(action):
561 action(None)
562 return
563
565 """ this actually does not make the folder. it has to be there """
566 self.folder = getattr(THREAD_LOCAL,'folder','')
567
568
569 if False and self.folder and not exists(self.folder):
570 os.mkdir(self.folder)
571
573 """hook for the after_connection parameter"""
574 if callable(self._after_connection):
575 self._after_connection(self)
576 self.after_connection()
577
579 """ this it is supposed to be overloaded by adapters"""
580 pass
581
583 """
584 this function defines: self.connection and self.cursor
585 (iff cursor is True)
586 if self.pool_size>0 it will try pull the connection from the pool
587 if the connection is not active (closed by db server) it will loop
588 if not self.pool_size or no active connections in pool makes a new one
589 """
590 if getattr(self,'connection',None) != None:
591 return
592 if f is None:
593 f = self.connector
594
595 if not self.pool_size:
596 self.connection = f()
597 self.cursor = cursor and self.connection.cursor()
598 else:
599 uri = self.uri
600 POOLS = ConnectionPool.POOLS
601 while True:
602 GLOBAL_LOCKER.acquire()
603 if not uri in POOLS:
604 POOLS[uri] = []
605 if POOLS[uri]:
606 self.connection = POOLS[uri].pop()
607 GLOBAL_LOCKER.release()
608 self.cursor = cursor and self.connection.cursor()
609 try:
610 if self.cursor and self.check_active_connection:
611 self.execute('SELECT 1;')
612 break
613 except:
614 pass
615 else:
616 GLOBAL_LOCKER.release()
617 self.connection = f()
618 self.cursor = cursor and self.connection.cursor()
619 break
620 self.after_connection_hook()
621
628 native_json = False
629 driver = None
630 driver_name = None
631 drivers = ()
632 connection = None
633 maxcharlength = MAXCHARLENGTH
634 commit_on_alter_table = False
635 support_distributed_transaction = False
636 uploads_in_blob = False
637 can_select_for_update = True
638
639 TRUE = 'T'
640 FALSE = 'F'
641 types = {
642 'boolean': 'CHAR(1)',
643 'string': 'CHAR(%(length)s)',
644 'text': 'TEXT',
645 'json': 'TEXT',
646 'password': 'CHAR(%(length)s)',
647 'blob': 'BLOB',
648 'upload': 'CHAR(%(length)s)',
649 'integer': 'INTEGER',
650 'bigint': 'INTEGER',
651 'float':'DOUBLE',
652 'double': 'DOUBLE',
653 'decimal': 'DOUBLE',
654 'date': 'DATE',
655 'time': 'TIME',
656 'datetime': 'TIMESTAMP',
657 'id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
658 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
659 'list:integer': 'TEXT',
660 'list:string': 'TEXT',
661 'list:reference': 'TEXT',
662
663 'big-id': 'BIGINT PRIMARY KEY AUTOINCREMENT',
664 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
665 }
666
668 return table._id != None
669
671 return "'%s'" % obj.replace("'", "''")
672
674 if isinstance(obj,(int,float)):
675 return str(obj)
676 return self.adapt(str(obj))
677
679 return self.driver.IntegrityError
680
682 return self.driver.OperationalError
683
685 """
686 to be used ONLY for files that on GAE may not be on filesystem
687 """
688 return exists(filename)
689
690 - def file_open(self, filename, mode='rb', lock=True):
691 """
692 to be used ONLY for files that on GAE may not be on filesystem
693 """
694 if have_portalocker and lock:
695 fileobj = portalocker.LockedFile(filename,mode)
696 else:
697 fileobj = open(filename,mode)
698 return fileobj
699
701 """
702 to be used ONLY for files that on GAE may not be on filesystem
703 """
704 if fileobj:
705 fileobj.close()
706
709
711 if getattr(self,'driver',None) != None:
712 return
713 drivers_available = [driver for driver in self.drivers
714 if driver in globals()]
715 if uri:
716 items = uri.split('://',1)[0].split(':')
717 request_driver = items[1] if len(items)>1 else None
718 else:
719 request_driver = None
720 request_driver = request_driver or adapter_args.get('driver')
721 if request_driver:
722 if request_driver in drivers_available:
723 self.driver_name = request_driver
724 self.driver = globals().get(request_driver)
725 else:
726 raise RuntimeError("driver %s not available" % request_driver)
727 elif drivers_available:
728 self.driver_name = drivers_available[0]
729 self.driver = globals().get(self.driver_name)
730 else:
731 raise RuntimeError("no driver available %s" % str(self.drivers))
732
733
734 - def __init__(self, db,uri,pool_size=0, folder=None, db_codec='UTF-8',
735 credential_decoder=IDENTITY, driver_args={},
736 adapter_args={},do_connect=True, after_connection=None):
737 self.db = db
738 self.dbengine = "None"
739 self.uri = uri
740 self.pool_size = pool_size
741 self.folder = folder
742 self.db_codec = db_codec
743 self._after_connection = after_connection
744 class Dummy(object):
745 lastrowid = 1
746 def __getattr__(self, value):
747 return lambda *a, **b: []
748 self.connection = Dummy()
749 self.cursor = Dummy()
750
752 return '%s_sequence' % tablename
753
755 return '%s_sequence' % tablename
756
759
760 - def create_table(self, table,
761 migrate=True,
762 fake_migrate=False,
763 polymodel=None):
764 db = table._db
765 fields = []
766
767 postcreation_fields = []
768 sql_fields = {}
769 sql_fields_aux = {}
770 TFK = {}
771 tablename = table._tablename
772 sortable = 0
773 types = self.types
774 for field in table:
775 sortable += 1
776 field_name = field.name
777 field_type = field.type
778 if isinstance(field_type,SQLCustomType):
779 ftype = field_type.native or field_type.type
780 elif field_type.startswith('reference'):
781 referenced = field_type[10:].strip()
782 if referenced == '.':
783 referenced = tablename
784 constraint_name = self.constraint_name(tablename, field_name)
785 if not '.' in referenced \
786 and referenced != tablename \
787 and hasattr(table,'_primarykey'):
788 ftype = types['integer']
789 else:
790 if hasattr(table,'_primarykey'):
791 rtablename,rfieldname = referenced.split('.')
792 rtable = db[rtablename]
793 rfield = rtable[rfieldname]
794
795 if rfieldname in rtable._primarykey or \
796 rfield.unique:
797 ftype = types[rfield.type[:9]] % \
798 dict(length=rfield.length)
799
800 if not rfield.unique and len(rtable._primarykey)>1:
801
802 if rtablename not in TFK:
803 TFK[rtablename] = {}
804 TFK[rtablename][rfieldname] = field_name
805 else:
806 ftype = ftype + \
807 types['reference FK'] % dict(
808 constraint_name = constraint_name,
809 foreign_key = '%s (%s)' % (rtablename,
810 rfieldname),
811 table_name = tablename,
812 field_name = field_name,
813 on_delete_action=field.ondelete)
814 else:
815
816 if referenced in db:
817 id_fieldname = db[referenced]._id.name
818 elif referenced == tablename:
819 id_fieldname = table._id.name
820 else:
821 id_fieldname = 'id'
822 ftype = types[field_type[:9]] % dict(
823 index_name = field_name+'__idx',
824 field_name = field_name,
825 constraint_name = constraint_name,
826 foreign_key = '%s (%s)' % (referenced,
827 id_fieldname),
828 on_delete_action=field.ondelete)
829 elif field_type.startswith('list:reference'):
830 ftype = types[field_type[:14]]
831 elif field_type.startswith('decimal'):
832 precision, scale = map(int,field_type[8:-1].split(','))
833 ftype = types[field_type[:7]] % \
834 dict(precision=precision,scale=scale)
835 elif field_type.startswith('geo'):
836 if not hasattr(self,'srid'):
837 raise RuntimeError('Adapter does not support geometry')
838 srid = self.srid
839 geotype, parms = field_type[:-1].split('(')
840 if not geotype in types:
841 raise SyntaxError(
842 'Field: unknown field type: %s for %s' \
843 % (field_type, field_name))
844 ftype = types[geotype]
845 if self.dbengine == 'postgres' and geotype == 'geometry':
846
847 dimension = 2
848 parms = parms.split(',')
849 if len(parms) == 3:
850 schema, srid, dimension = parms
851 elif len(parms) == 2:
852 schema, srid = parms
853 else:
854 schema = parms[0]
855 ftype = "SELECT AddGeometryColumn ('%%(schema)s', '%%(tablename)s', '%%(fieldname)s', %%(srid)s, '%s', %%(dimension)s);" % types[geotype]
856 ftype = ftype % dict(schema=schema,
857 tablename=tablename,
858 fieldname=field_name, srid=srid,
859 dimension=dimension)
860 postcreation_fields.append(ftype)
861 elif not field_type in types:
862 raise SyntaxError('Field: unknown field type: %s for %s' % \
863 (field_type, field_name))
864 else:
865 ftype = types[field_type]\
866 % dict(length=field.length)
867 if not field_type.startswith('id') and \
868 not field_type.startswith('reference'):
869 if field.notnull:
870 ftype += ' NOT NULL'
871 else:
872 ftype += self.ALLOW_NULL()
873 if field.unique:
874 ftype += ' UNIQUE'
875 if field.custom_qualifier:
876 ftype += ' %s' % field.custom_qualifier
877
878
879 sql_fields[field_name] = dict(
880 length=field.length,
881 unique=field.unique,
882 notnull=field.notnull,
883 sortable=sortable,
884 type=str(field_type),
885 sql=ftype)
886
887 if field.notnull and not field.default is None:
888
889
890
891
892
893
894 not_null = self.NOT_NULL(field.default, field_type)
895 ftype = ftype.replace('NOT NULL', not_null)
896 sql_fields_aux[field_name] = dict(sql=ftype)
897
898
899 if not (self.dbengine == 'postgres' and \
900 field_type.startswith('geom')):
901 fields.append('%s %s' % (field_name, ftype))
902 other = ';'
903
904
905 if self.dbengine == 'mysql':
906 if not hasattr(table, "_primarykey"):
907 fields.append('PRIMARY KEY(%s)' % table._id.name)
908 other = ' ENGINE=InnoDB CHARACTER SET utf8;'
909
910 fields = ',\n '.join(fields)
911 for rtablename in TFK:
912 rfields = TFK[rtablename]
913 pkeys = db[rtablename]._primarykey
914 fkeys = [ rfields[k] for k in pkeys ]
915 fields = fields + ',\n ' + \
916 types['reference TFK'] % dict(
917 table_name = tablename,
918 field_name=', '.join(fkeys),
919 foreign_table = rtablename,
920 foreign_key = ', '.join(pkeys),
921 on_delete_action = field.ondelete)
922
923 if hasattr(table,'_primarykey'):
924 query = "CREATE TABLE %s(\n %s,\n %s) %s" % \
925 (tablename, fields,
926 self.PRIMARY_KEY(', '.join(table._primarykey)),other)
927 else:
928 query = "CREATE TABLE %s(\n %s\n)%s" % \
929 (tablename, fields, other)
930
931 if self.uri.startswith('sqlite:///') \
932 or self.uri.startswith('spatialite:///'):
933 path_encoding = sys.getfilesystemencoding() \
934 or locale.getdefaultlocale()[1] or 'utf8'
935 dbpath = self.uri[9:self.uri.rfind('/')]\
936 .decode('utf8').encode(path_encoding)
937 else:
938 dbpath = self.folder
939
940 if not migrate:
941 return query
942 elif self.uri.startswith('sqlite:memory')\
943 or self.uri.startswith('spatialite:memory'):
944 table._dbt = None
945 elif isinstance(migrate, str):
946 table._dbt = pjoin(dbpath, migrate)
947 else:
948 table._dbt = pjoin(
949 dbpath, '%s_%s.table' % (table._db._uri_hash, tablename))
950
951 if table._dbt:
952 table._loggername = pjoin(dbpath, 'sql.log')
953 logfile = self.file_open(table._loggername, 'a')
954 else:
955 logfile = None
956 if not table._dbt or not self.file_exists(table._dbt):
957 if table._dbt:
958 logfile.write('timestamp: %s\n'
959 % datetime.datetime.today().isoformat())
960 logfile.write(query + '\n')
961 if not fake_migrate:
962 self.create_sequence_and_triggers(query,table)
963 table._db.commit()
964
965
966 for query in postcreation_fields:
967 self.execute(query)
968 table._db.commit()
969 if table._dbt:
970 tfile = self.file_open(table._dbt, 'w')
971 pickle.dump(sql_fields, tfile)
972 self.file_close(tfile)
973 if fake_migrate:
974 logfile.write('faked!\n')
975 else:
976 logfile.write('success!\n')
977 else:
978 tfile = self.file_open(table._dbt, 'r')
979 try:
980 sql_fields_old = pickle.load(tfile)
981 except EOFError:
982 self.file_close(tfile)
983 self.file_close(logfile)
984 raise RuntimeError('File %s appears corrupted' % table._dbt)
985 self.file_close(tfile)
986 if sql_fields != sql_fields_old:
987 self.migrate_table(table,
988 sql_fields, sql_fields_old,
989 sql_fields_aux, logfile,
990 fake_migrate=fake_migrate)
991 self.file_close(logfile)
992 return query
993
994 - def migrate_table(
995 self,
996 table,
997 sql_fields,
998 sql_fields_old,
999 sql_fields_aux,
1000 logfile,
1001 fake_migrate=False,
1002 ):
1003 db = table._db
1004 db._migrated.append(table._tablename)
1005 tablename = table._tablename
1006 def fix(item):
1007 k,v=item
1008 if not isinstance(v,dict):
1009 v=dict(type='unkown',sql=v)
1010 return k.lower(),v
1011
1012
1013 sql_fields = dict(map(fix,sql_fields.iteritems()))
1014 sql_fields_old = dict(map(fix,sql_fields_old.iteritems()))
1015 sql_fields_aux = dict(map(fix,sql_fields_aux.iteritems()))
1016 if db._debug:
1017 logging.debug('migrating %s to %s' % (sql_fields_old,sql_fields))
1018
1019 keys = sql_fields.keys()
1020 for key in sql_fields_old:
1021 if not key in keys:
1022 keys.append(key)
1023 new_add = self.concat_add(tablename)
1024
1025 metadata_change = False
1026 sql_fields_current = copy.copy(sql_fields_old)
1027 for key in keys:
1028 query = None
1029 if not key in sql_fields_old:
1030 sql_fields_current[key] = sql_fields[key]
1031 if self.dbengine in ('postgres',) and \
1032 sql_fields[key]['type'].startswith('geometry'):
1033
1034 query = [ sql_fields[key]['sql'] ]
1035 else:
1036 query = ['ALTER TABLE %s ADD %s %s;' % \
1037 (tablename, key,
1038 sql_fields_aux[key]['sql'].replace(', ', new_add))]
1039 metadata_change = True
1040 elif self.dbengine in ('sqlite', 'spatialite'):
1041 if key in sql_fields:
1042 sql_fields_current[key] = sql_fields[key]
1043 metadata_change = True
1044 elif not key in sql_fields:
1045 del sql_fields_current[key]
1046 ftype = sql_fields_old[key]['type']
1047 if self.dbengine in ('postgres',) \
1048 and ftype.startswith('geometry'):
1049 geotype, parms = ftype[:-1].split('(')
1050 schema = parms.split(',')[0]
1051 query = [ "SELECT DropGeometryColumn ('%(schema)s', '%(table)s', '%(field)s');" % dict(schema=schema, table=tablename, field=key,) ]
1052 elif not self.dbengine in ('firebird',):
1053 query = ['ALTER TABLE %s DROP COLUMN %s;'
1054 % (tablename, key)]
1055 else:
1056 query = ['ALTER TABLE %s DROP %s;' % (tablename, key)]
1057 metadata_change = True
1058 elif sql_fields[key]['sql'] != sql_fields_old[key]['sql'] \
1059 and not (key in table.fields and
1060 isinstance(table[key].type, SQLCustomType)) \
1061 and not sql_fields[key]['type'].startswith('reference')\
1062 and not sql_fields[key]['type'].startswith('double')\
1063 and not sql_fields[key]['type'].startswith('id'):
1064 sql_fields_current[key] = sql_fields[key]
1065 t = tablename
1066 tt = sql_fields_aux[key]['sql'].replace(', ', new_add)
1067 if not self.dbengine in ('firebird',):
1068 query = ['ALTER TABLE %s ADD %s__tmp %s;' % (t, key, tt),
1069 'UPDATE %s SET %s__tmp=%s;' % (t, key, key),
1070 'ALTER TABLE %s DROP COLUMN %s;' % (t, key),
1071 'ALTER TABLE %s ADD %s %s;' % (t, key, tt),
1072 'UPDATE %s SET %s=%s__tmp;' % (t, key, key),
1073 'ALTER TABLE %s DROP COLUMN %s__tmp;' % (t, key)]
1074 else:
1075 query = ['ALTER TABLE %s ADD %s__tmp %s;' % (t, key, tt),
1076 'UPDATE %s SET %s__tmp=%s;' % (t, key, key),
1077 'ALTER TABLE %s DROP %s;' % (t, key),
1078 'ALTER TABLE %s ADD %s %s;' % (t, key, tt),
1079 'UPDATE %s SET %s=%s__tmp;' % (t, key, key),
1080 'ALTER TABLE %s DROP %s__tmp;' % (t, key)]
1081 metadata_change = True
1082 elif sql_fields[key]['type'] != sql_fields_old[key]['type']:
1083 sql_fields_current[key] = sql_fields[key]
1084 metadata_change = True
1085
1086 if query:
1087 logfile.write('timestamp: %s\n'
1088 % datetime.datetime.today().isoformat())
1089 db['_lastsql'] = '\n'.join(query)
1090 for sub_query in query:
1091 logfile.write(sub_query + '\n')
1092 if not fake_migrate:
1093 self.execute(sub_query)
1094
1095
1096
1097 if db._adapter.commit_on_alter_table:
1098 db.commit()
1099 tfile = self.file_open(table._dbt, 'w')
1100 pickle.dump(sql_fields_current, tfile)
1101 self.file_close(tfile)
1102 logfile.write('success!\n')
1103 else:
1104 logfile.write('faked!\n')
1105 elif metadata_change:
1106 tfile = self.file_open(table._dbt, 'w')
1107 pickle.dump(sql_fields_current, tfile)
1108 self.file_close(tfile)
1109
1110 if metadata_change and \
1111 not (query and self.dbengine in ('mysql','oracle','firebird')):
1112 db.commit()
1113 tfile = self.file_open(table._dbt, 'w')
1114 pickle.dump(sql_fields_current, tfile)
1115 self.file_close(tfile)
1116
1117 - def LOWER(self, first):
1119
1120 - def UPPER(self, first):
1122
1123 - def COUNT(self, first, distinct=None):
1124 return ('COUNT(%s)' if not distinct else 'COUNT(DISTINCT %s)') \
1125 % self.expand(first)
1126
1128 return "EXTRACT(%s FROM %s)" % (what, self.expand(first))
1129
1130 - def EPOCH(self, first):
1132
1135
1138
1141
1144
1145 - def NOT_NULL(self, default, field_type):
1146 return 'NOT NULL DEFAULT %s' % self.represent(default,field_type)
1147
1149 expressions = [self.expand(first)]+[self.expand(e) for e in second]
1150 return 'COALESCE(%s)' % ','.join(expressions)
1151
1154
1155 - def RAW(self, first):
1157
1160
1162 return 'SUBSTR(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
1163
1165 return 'PRIMARY KEY(%s)' % key
1166
1167 - def _drop(self, table, mode):
1168 return ['DROP TABLE %s;' % table]
1169
1170 - def drop(self, table, mode=''):
1186
1187 - def _insert(self, table, fields):
1194
1196 return 'INSERT INTO %s DEFAULT VALUES;' % table
1197
1198 - def insert(self, table, fields):
1199 query = self._insert(table,fields)
1200 try:
1201 self.execute(query)
1202 except Exception:
1203 e = sys.exc_info()[1]
1204 if isinstance(e,self.integrity_error_class()):
1205 return None
1206 raise e
1207 if hasattr(table,'_primarykey'):
1208 return dict([(k[0].name, k[1]) for k in fields \
1209 if k[0].name in table._primarykey])
1210 id = self.lastrowid(table)
1211 if not isinstance(id,int):
1212 return id
1213 rid = Reference(id)
1214 (rid._table, rid._record) = (table, None)
1215 return rid
1216
1218 return [self.insert(table,item) for item in items]
1219
1220 - def NOT(self, first):
1222
1223 - def AND(self, first, second):
1225
1226 - def OR(self, first, second):
1228
1229 - def BELONGS(self, first, second):
1230 if isinstance(second, str):
1231 return '(%s IN (%s))' % (self.expand(first), second[:-1])
1232 elif not second:
1233 return '(1=0)'
1234 items = ','.join(self.expand(item, first.type) for item in second)
1235 return '(%s IN (%s))' % (self.expand(first), items)
1236
1237 - def REGEXP(self, first, second):
1238 "regular expression operator"
1239 raise NotImplementedError
1240
1241 - def LIKE(self, first, second):
1242 "case sensitive like operator"
1243 raise NotImplementedError
1244
1245 - def ILIKE(self, first, second):
1246 "case in-sensitive like operator"
1247 return '(%s LIKE %s)' % (self.expand(first),
1248 self.expand(second, 'string'))
1249
1251 return '(%s LIKE %s)' % (self.expand(first),
1252 self.expand(second+'%', 'string'))
1253
1255 return '(%s LIKE %s)' % (self.expand(first),
1256 self.expand('%'+second, 'string'))
1257
1258 - def CONTAINS(self, first, second, case_sensitive=False):
1259 if isinstance(second,Expression):
1260 field = self.expand(first)
1261 expr = self.expand(second,'string')
1262 if first.type.startswith('list:'):
1263 expr = 'CONCAT("|", %s, "|")' % expr
1264 elif not first.type in ('string', 'text', 'json'):
1265 raise RuntimeError("Expression Not Supported")
1266 return 'INSTR(%s,%s)' % (field, expr)
1267 else:
1268 if first.type in ('string', 'text', 'json'):
1269 key = '%'+str(second).replace('%','%%')+'%'
1270 elif first.type.startswith('list:'):
1271 key = '%|'+str(second).replace('|','||').replace('%','%%')+'|%'
1272 else:
1273 raise RuntimeError("Expression Not Supported")
1274 op = case_sensitive and self.LIKE or self.ILIKE
1275 return op(first,key)
1276
1277 - def EQ(self, first, second=None):
1282
1283 - def NE(self, first, second=None):
1288
1289 - def LT(self,first,second=None):
1290 if second is None:
1291 raise RuntimeError("Cannot compare %s < None" % first)
1292 return '(%s < %s)' % (self.expand(first),
1293 self.expand(second,first.type))
1294
1295 - def LE(self,first,second=None):
1296 if second is None:
1297 raise RuntimeError("Cannot compare %s <= None" % first)
1298 return '(%s <= %s)' % (self.expand(first),
1299 self.expand(second,first.type))
1300
1301 - def GT(self,first,second=None):
1302 if second is None:
1303 raise RuntimeError("Cannot compare %s > None" % first)
1304 return '(%s > %s)' % (self.expand(first),
1305 self.expand(second,first.type))
1306
1307 - def GE(self,first,second=None):
1308 if second is None:
1309 raise RuntimeError("Cannot compare %s >= None" % first)
1310 return '(%s >= %s)' % (self.expand(first),
1311 self.expand(second,first.type))
1312
1313 - def ADD(self, first, second):
1316
1317 - def SUB(self, first, second):
1320
1321 - def MUL(self, first, second):
1324
1325 - def DIV(self, first, second):
1328
1329 - def MOD(self, first, second):
1332
1333 - def AS(self, first, second):
1335
1336 - def ON(self, first, second):
1340
1343
1344 - def COMMA(self, first, second):
1346
1347 - def expand(self, expression, field_type=None):
1348 if isinstance(expression, Field):
1349 return '%s.%s' % (expression.tablename, expression.name)
1350 elif isinstance(expression, (Expression, Query)):
1351 first = expression.first
1352 second = expression.second
1353 op = expression.op
1354 optional_args = expression.optional_args or {}
1355 if not second is None:
1356 return op(first, second, **optional_args)
1357 elif not first is None:
1358 return op(first,**optional_args)
1359 elif isinstance(op, str):
1360 if op.endswith(';'):
1361 op=op[:-1]
1362 return '(%s)' % op
1363 else:
1364 return op()
1365 elif field_type:
1366 return str(self.represent(expression,field_type))
1367 elif isinstance(expression,(list,tuple)):
1368 return ','.join(self.represent(item,field_type) \
1369 for item in expression)
1370 elif isinstance(expression, bool):
1371 return '1' if expression else '0'
1372 else:
1373 return str(expression)
1374
1375 - def alias(self, table, alias):
1376 """
1377 Given a table object, makes a new table object
1378 with alias name.
1379 """
1380 other = copy.copy(table)
1381 other['_ot'] = other._tablename
1382 other['ALL'] = SQLALL(other)
1383 other['_tablename'] = alias
1384 for fieldname in other.fields:
1385 other[fieldname] = copy.copy(other[fieldname])
1386 other[fieldname]._tablename = alias
1387 other[fieldname].tablename = alias
1388 other[fieldname].table = other
1389 table._db[alias] = other
1390 return other
1391
1393 tablename = table._tablename
1394 return ['TRUNCATE TABLE %s %s;' % (tablename, mode or '')]
1395
1397
1398 if table._dbt:
1399 logfile = self.file_open(table._loggername, 'a')
1400 else:
1401 class Logfile(object):
1402 def write(self, value):
1403 pass
1404 def close(self):
1405 pass
1406 logfile = Logfile()
1407
1408 try:
1409 queries = table._db._adapter._truncate(table, mode)
1410 for query in queries:
1411 logfile.write(query + '\n')
1412 self.execute(query)
1413 table._db.commit()
1414 logfile.write('success!\n')
1415 finally:
1416 logfile.close()
1417
1418 - def _update(self, tablename, query, fields):
1419 if query:
1420 if use_common_filters(query):
1421 query = self.common_filter(query, [tablename])
1422 sql_w = ' WHERE ' + self.expand(query)
1423 else:
1424 sql_w = ''
1425 sql_v = ','.join(['%s=%s' % (field.name,
1426 self.expand(value, field.type)) \
1427 for (field, value) in fields])
1428 return 'UPDATE %s SET %s%s;' % (tablename, sql_v, sql_w)
1429
1430 - def update(self, tablename, query, fields):
1431 sql = self._update(tablename, query, fields)
1432 self.execute(sql)
1433 try:
1434 return self.cursor.rowcount
1435 except:
1436 return None
1437
1438 - def _delete(self, tablename, query):
1439 if query:
1440 if use_common_filters(query):
1441 query = self.common_filter(query, [tablename])
1442 sql_w = ' WHERE ' + self.expand(query)
1443 else:
1444 sql_w = ''
1445 return 'DELETE FROM %s%s;' % (tablename, sql_w)
1446
1447 - def delete(self, tablename, query):
1448 sql = self._delete(tablename, query)
1449
1450 db = self.db
1451 table = db[tablename]
1452 if self.dbengine in ('sqlite', 'spatialite') and table._referenced_by:
1453 deleted = [x[table._id.name] for x in db(query).select(table._id)]
1454
1455 self.execute(sql)
1456 try:
1457 counter = self.cursor.rowcount
1458 except:
1459 counter = None
1460
1461 if self.dbengine in ('sqlite', 'spatialite') and counter:
1462 for field in table._referenced_by:
1463 if field.type=='reference '+table._tablename \
1464 and field.ondelete=='CASCADE':
1465 db(field.belongs(deleted)).delete()
1466
1467 return counter
1468
1470 tablenames = self.tables(query)
1471 if len(tablenames)==1:
1472 return tablenames[0]
1473 elif len(tablenames)<1:
1474 raise RuntimeError("No table selected")
1475 else:
1476 raise RuntimeError("Too many tables selected")
1477
1479 db = self.db
1480 new_fields = []
1481 append = new_fields.append
1482 for item in fields:
1483 if isinstance(item,SQLALL):
1484 new_fields += item._table
1485 elif isinstance(item,str):
1486 if REGEX_TABLE_DOT_FIELD.match(item):
1487 tablename,fieldname = item.split('.')
1488 append(db[tablename][fieldname])
1489 else:
1490 append(Expression(db,lambda item=item:item))
1491 else:
1492 append(item)
1493
1494 if not new_fields:
1495 for table in tablenames:
1496 for field in db[table]:
1497 append(field)
1498 return new_fields
1499
1500 - def _select(self, query, fields, attributes):
1501 tables = self.tables
1502 for key in set(attributes.keys())-SELECT_ARGS:
1503 raise SyntaxError('invalid select attribute: %s' % key)
1504 args_get = attributes.get
1505 tablenames = tables(query)
1506 for field in fields:
1507 if isinstance(field, basestring) \
1508 and REGEX_TABLE_DOT_FIELD.match(field):
1509 tn,fn = field.split('.')
1510 field = self.db[tn][fn]
1511 for tablename in tables(field):
1512 if not tablename in tablenames:
1513 tablenames.append(tablename)
1514
1515 if len(tablenames) < 1:
1516 raise SyntaxError('Set: no tables selected')
1517 self._colnames = map(self.expand, fields)
1518 def geoexpand(field):
1519 if isinstance(field.type,str) and field.type.startswith('geometry'):
1520 field = field.st_astext()
1521 return self.expand(field)
1522 sql_f = ', '.join(map(geoexpand, fields))
1523 sql_o = ''
1524 sql_s = ''
1525 left = args_get('left', False)
1526 inner_join = args_get('join', False)
1527 distinct = args_get('distinct', False)
1528 groupby = args_get('groupby', False)
1529 orderby = args_get('orderby', False)
1530 having = args_get('having', False)
1531 limitby = args_get('limitby', False)
1532 for_update = args_get('for_update', False)
1533 if self.can_select_for_update is False and for_update is True:
1534 raise SyntaxError('invalid select attribute: for_update')
1535 if distinct is True:
1536 sql_s += 'DISTINCT'
1537 elif distinct:
1538 sql_s += 'DISTINCT ON (%s)' % distinct
1539 if inner_join:
1540 icommand = self.JOIN()
1541 if not isinstance(inner_join, (tuple, list)):
1542 inner_join = [inner_join]
1543 ijoint = [t._tablename for t in inner_join
1544 if not isinstance(t,Expression)]
1545 ijoinon = [t for t in inner_join if isinstance(t, Expression)]
1546 itables_to_merge={}
1547 [itables_to_merge.update(
1548 dict.fromkeys(tables(t))) for t in ijoinon]
1549 ijoinont = [t.first._tablename for t in ijoinon]
1550 [itables_to_merge.pop(t) for t in ijoinont
1551 if t in itables_to_merge]
1552 iimportant_tablenames = ijoint + ijoinont + itables_to_merge.keys()
1553 iexcluded = [t for t in tablenames
1554 if not t in iimportant_tablenames]
1555 if left:
1556 join = attributes['left']
1557 command = self.LEFT_JOIN()
1558 if not isinstance(join, (tuple, list)):
1559 join = [join]
1560 joint = [t._tablename for t in join
1561 if not isinstance(t, Expression)]
1562 joinon = [t for t in join if isinstance(t, Expression)]
1563
1564 tables_to_merge={}
1565 [tables_to_merge.update(
1566 dict.fromkeys(tables(t))) for t in joinon]
1567 joinont = [t.first._tablename for t in joinon]
1568 [tables_to_merge.pop(t) for t in joinont if t in tables_to_merge]
1569 important_tablenames = joint + joinont + tables_to_merge.keys()
1570 excluded = [t for t in tablenames
1571 if not t in important_tablenames ]
1572 else:
1573 excluded = tablenames
1574
1575 if use_common_filters(query):
1576 query = self.common_filter(query,excluded)
1577 sql_w = ' WHERE ' + self.expand(query) if query else ''
1578
1579 def alias(t):
1580 return str(self.db[t])
1581 if inner_join and not left:
1582 sql_t = ', '.join([alias(t) for t in iexcluded + \
1583 itables_to_merge.keys()])
1584 for t in ijoinon:
1585 sql_t += ' %s %s' % (icommand, str(t))
1586 elif not inner_join and left:
1587 sql_t = ', '.join([alias(t) for t in excluded + \
1588 tables_to_merge.keys()])
1589 if joint:
1590 sql_t += ' %s %s' % (command, ','.join([t for t in joint]))
1591 for t in joinon:
1592 sql_t += ' %s %s' % (command, str(t))
1593 elif inner_join and left:
1594 all_tables_in_query = set(important_tablenames + \
1595 iimportant_tablenames + \
1596 tablenames)
1597 tables_in_joinon = set(joinont + ijoinont)
1598 tables_not_in_joinon = \
1599 all_tables_in_query.difference(tables_in_joinon)
1600 sql_t = ','.join([alias(t) for t in tables_not_in_joinon])
1601 for t in ijoinon:
1602 sql_t += ' %s %s' % (icommand, str(t))
1603 if joint:
1604 sql_t += ' %s %s' % (command, ','.join([t for t in joint]))
1605 for t in joinon:
1606 sql_t += ' %s %s' % (command, str(t))
1607 else:
1608 sql_t = ', '.join(alias(t) for t in tablenames)
1609 if groupby:
1610 if isinstance(groupby, (list, tuple)):
1611 groupby = xorify(groupby)
1612 sql_o += ' GROUP BY %s' % self.expand(groupby)
1613 if having:
1614 sql_o += ' HAVING %s' % attributes['having']
1615 if orderby:
1616 if isinstance(orderby, (list, tuple)):
1617 orderby = xorify(orderby)
1618 if str(orderby) == '<random>':
1619 sql_o += ' ORDER BY %s' % self.RANDOM()
1620 else:
1621 sql_o += ' ORDER BY %s' % self.expand(orderby)
1622 if limitby:
1623 if not orderby and tablenames:
1624 sql_o += ' ORDER BY %s' % ', '.join(['%s.%s'%(t,x) for t in tablenames for x in (hasattr(self.db[t],'_primarykey') and self.db[t]._primarykey or [self.db[t]._id.name])])
1625
1626 sql = self.select_limitby(sql_s, sql_f, sql_t, sql_w, sql_o, limitby)
1627 if for_update and self.can_select_for_update is True:
1628 sql = sql.rstrip(';') + ' FOR UPDATE;'
1629 return sql
1630
1631 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
1632 if limitby:
1633 (lmin, lmax) = limitby
1634 sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
1635 return 'SELECT %s %s FROM %s%s%s;' % \
1636 (sql_s, sql_f, sql_t, sql_w, sql_o)
1637
1639 return self.cursor.fetchall()
1640
1642 args_get = attributes.get
1643 cache = args_get('cache',None)
1644 if not cache:
1645 self.execute(sql)
1646 rows = self._fetchall()
1647 else:
1648 (cache_model, time_expire) = cache
1649 key = self.uri + '/' + sql + '/rows'
1650 if len(key)>200: key = hashlib_md5(key).hexdigest()
1651 def _select_aux2():
1652 self.execute(sql)
1653 return self._fetchall()
1654 rows = cache_model(key,_select_aux2,time_expire)
1655 if isinstance(rows,tuple):
1656 rows = list(rows)
1657 limitby = args_get('limitby', None) or (0,)
1658 rows = self.rowslice(rows,limitby[0],None)
1659 processor = args_get('processor',self.parse)
1660 cacheable = args_get('cacheable',False)
1661 return processor(rows,fields,self._colnames,cacheable=cacheable)
1662
1663 - def select(self, query, fields, attributes):
1664 """
1665 Always returns a Rows object, possibly empty.
1666 """
1667 sql = self._select(query, fields, attributes)
1668 cache = attributes.get('cache', None)
1669 if cache and attributes.get('cacheable',False):
1670 del attributes['cache']
1671 (cache_model, time_expire) = cache
1672 key = self.uri + '/' + sql
1673 if len(key)>200: key = hashlib_md5(key).hexdigest()
1674 args = (sql,fields,attributes)
1675 return cache_model(
1676 key,
1677 lambda self=self,args=args:self._select_aux(*args),
1678 time_expire)
1679 else:
1680 return self._select_aux(sql,fields,attributes)
1681
1682 - def _count(self, query, distinct=None):
1683 tablenames = self.tables(query)
1684 if query:
1685 if use_common_filters(query):
1686 query = self.common_filter(query, tablenames)
1687 sql_w = ' WHERE ' + self.expand(query)
1688 else:
1689 sql_w = ''
1690 sql_t = ','.join(tablenames)
1691 if distinct:
1692 if isinstance(distinct,(list, tuple)):
1693 distinct = xorify(distinct)
1694 sql_d = self.expand(distinct)
1695 return 'SELECT count(DISTINCT %s) FROM %s%s;' % \
1696 (sql_d, sql_t, sql_w)
1697 return 'SELECT count(*) FROM %s%s;' % (sql_t, sql_w)
1698
1699 - def count(self, query, distinct=None):
1700 self.execute(self._count(query, distinct))
1701 return self.cursor.fetchone()[0]
1702
1714
1717
1720
1723
1726
1729
1732
1735
1738
1740 return '%s_%s__constraint' % (table,fieldname)
1741
1744
1746 if not self.connection: return None
1747 command = a[0]
1748 if self.db._debug:
1749 LOGGER.debug('SQL: %s' % command)
1750 self.db._lastsql = command
1751 t0 = time.time()
1752 ret = self.cursor.execute(*a, **b)
1753 self.db._timings.append((command,time.time()-t0))
1754 del self.db._timings[:-TIMINGSSIZE]
1755 return ret
1756
1759
1761 field_is_type = fieldtype.startswith
1762 if isinstance(obj, CALLABLETYPES):
1763 obj = obj()
1764 if isinstance(fieldtype, SQLCustomType):
1765 value = fieldtype.encoder(obj)
1766 if fieldtype.type in ('string','text', 'json'):
1767 return self.adapt(value)
1768 return value
1769 if isinstance(obj, (Expression, Field)):
1770 return str(obj)
1771 if field_is_type('list:'):
1772 if not obj:
1773 obj = []
1774 elif not isinstance(obj, (list, tuple)):
1775 obj = [obj]
1776 if field_is_type('list:string'):
1777 obj = map(str,obj)
1778 else:
1779 obj = map(int,obj)
1780
1781 if isinstance(obj, (list, tuple)) and (not fieldtype == "json"):
1782 obj = bar_encode(obj)
1783 if obj is None:
1784 return 'NULL'
1785 if obj == '' and not fieldtype[:2] in ['st', 'te', 'js', 'pa', 'up']:
1786 return 'NULL'
1787 r = self.represent_exceptions(obj, fieldtype)
1788 if not r is None:
1789 return r
1790 if fieldtype == 'boolean':
1791 if obj and not str(obj)[:1].upper() in '0F':
1792 return self.smart_adapt(self.TRUE)
1793 else:
1794 return self.smart_adapt(self.FALSE)
1795 if fieldtype == 'id' or fieldtype == 'integer':
1796 return str(int(obj))
1797 if field_is_type('decimal'):
1798 return str(obj)
1799 elif field_is_type('reference'):
1800 if fieldtype.find('.')>0:
1801 return repr(obj)
1802 elif isinstance(obj, (Row, Reference)):
1803 return str(obj['id'])
1804 return str(int(obj))
1805 elif fieldtype == 'double':
1806 return repr(float(obj))
1807 if isinstance(obj, unicode):
1808 obj = obj.encode(self.db_codec)
1809 if fieldtype == 'blob':
1810 obj = base64.b64encode(str(obj))
1811 elif fieldtype == 'date':
1812 if isinstance(obj, (datetime.date, datetime.datetime)):
1813 obj = obj.isoformat()[:10]
1814 else:
1815 obj = str(obj)
1816 elif fieldtype == 'datetime':
1817 if isinstance(obj, datetime.datetime):
1818 obj = obj.isoformat()[:19].replace('T',' ')
1819 elif isinstance(obj, datetime.date):
1820 obj = obj.isoformat()[:10]+' 00:00:00'
1821 else:
1822 obj = str(obj)
1823 elif fieldtype == 'time':
1824 if isinstance(obj, datetime.time):
1825 obj = obj.isoformat()[:10]
1826 else:
1827 obj = str(obj)
1828 elif fieldtype == 'json':
1829 if not self.native_json:
1830 if have_serializers:
1831 obj = serializers.json(obj)
1832 elif simplejson:
1833 obj = simplejson.dumps(items)
1834 else:
1835 raise RuntimeError("missing simplejson")
1836 if not isinstance(obj,bytes):
1837 obj = bytes(obj)
1838 try:
1839 obj.decode(self.db_codec)
1840 except:
1841 obj = obj.decode('latin1').encode(self.db_codec)
1842 return self.adapt(obj)
1843
1846
1849
1852
1853 - def rowslice(self, rows, minimum=0, maximum=None):
1854 """
1855 By default this function does nothing;
1856 overload when db does not do slicing.
1857 """
1858 return rows
1859
1860 - def parse_value(self, value, field_type, blob_decode=True):
1861 if field_type != 'blob' and isinstance(value, str):
1862 try:
1863 value = value.decode(self.db._db_codec)
1864 except Exception:
1865 pass
1866 if isinstance(value, unicode):
1867 value = value.encode('utf-8')
1868 if isinstance(field_type, SQLCustomType):
1869 value = field_type.decoder(value)
1870 if not isinstance(field_type, str) or value is None:
1871 return value
1872 elif field_type in ('string', 'text', 'password', 'upload', 'dict'):
1873 return value
1874 elif field_type.startswith('geo'):
1875 return value
1876 elif field_type == 'blob' and not blob_decode:
1877 return value
1878 else:
1879 key = REGEX_TYPE.match(field_type).group(0)
1880 return self.parsemap[key](value,field_type)
1881
1883 referee = field_type[10:].strip()
1884 if not '.' in referee:
1885 value = Reference(value)
1886 value._table, value._record = self.db[referee], None
1887 return value
1888
1890 return value == True or str(value)[:1].lower() == 't'
1891
1893 if isinstance(value, datetime.datetime):
1894 return value.date()
1895 if not isinstance(value, (datetime.date,datetime.datetime)):
1896 (y, m, d) = map(int, str(value)[:10].strip().split('-'))
1897 value = datetime.date(y, m, d)
1898 return value
1899
1901 if not isinstance(value, datetime.time):
1902 time_items = map(int,str(value)[:8].strip().split(':')[:3])
1903 if len(time_items) == 3:
1904 (h, mi, s) = time_items
1905 else:
1906 (h, mi, s) = time_items + [0]
1907 value = datetime.time(h, mi, s)
1908 return value
1909
1911 if not isinstance(value, datetime.datetime):
1912 value = str(value)
1913 date_part,time_part,timezone = value[:10],value[11:19],value[19:]
1914 if '+' in timezone:
1915 ms,tz = timezone.split('+')
1916 h,m = tz.split(':')
1917 dt = datetime.timedelta(seconds=3600*int(h)+60*int(m))
1918 elif '-' in timezone:
1919 ms,tz = timezone.split('-')
1920 h,m = tz.split(':')
1921 dt = -datetime.timedelta(seconds=3600*int(h)+60*int(m))
1922 else:
1923 dt = None
1924 (y, m, d) = map(int,date_part.split('-'))
1925 time_parts = time_part and time_part.split(':')[:3] or (0,0,0)
1926 while len(time_parts)<3: time_parts.append(0)
1927 time_items = map(int,time_parts)
1928 (h, mi, s) = time_items
1929 value = datetime.datetime(y, m, d, h, mi, s)
1930 if dt:
1931 value = value + dt
1932 return value
1933
1935 return base64.b64decode(str(value))
1936
1938 decimals = int(field_type[8:-1].split(',')[-1])
1939 if self.dbengine in ('sqlite', 'spatialite'):
1940 value = ('%.' + str(decimals) + 'f') % value
1941 if not isinstance(value, decimal.Decimal):
1942 value = decimal.Decimal(str(value))
1943 return value
1944
1949
1954
1959
1960 - def parse_id(self, value, field_type):
1962
1965
1968
1970 if not self.native_json:
1971 if not isinstance(value, basestring):
1972 raise RuntimeError('json data not a string')
1973 if isinstance(value, unicode):
1974 value = value.encode('utf-8')
1975 if have_serializers:
1976 value = serializers.loads_json(value)
1977 elif simplejson:
1978 value = simplejson.loads(value)
1979 else:
1980 raise RuntimeError("missing simplejson")
1981 return value
1982
1984 self.parsemap = {
1985 'id':self.parse_id,
1986 'integer':self.parse_integer,
1987 'bigint':self.parse_integer,
1988 'float':self.parse_double,
1989 'double':self.parse_double,
1990 'reference':self.parse_reference,
1991 'boolean':self.parse_boolean,
1992 'date':self.parse_date,
1993 'time':self.parse_time,
1994 'datetime':self.parse_datetime,
1995 'blob':self.parse_blob,
1996 'decimal':self.parse_decimal,
1997 'json':self.parse_json,
1998 'list:integer':self.parse_list_integers,
1999 'list:reference':self.parse_list_references,
2000 'list:string':self.parse_list_strings,
2001 }
2002
2003 - def parse(self, rows, fields, colnames, blob_decode=True,
2004 cacheable = False):
2005 self.build_parsemap()
2006 db = self.db
2007 virtualtables = []
2008 new_rows = []
2009 tmps = []
2010 for colname in colnames:
2011 if not REGEX_TABLE_DOT_FIELD.match(colname):
2012 tmps.append(None)
2013 else:
2014 (tablename, fieldname) = colname.split('.')
2015 table = db[tablename]
2016 field = table[fieldname]
2017 ft = field.type
2018 tmps.append((tablename,fieldname,table,field,ft))
2019 for (i,row) in enumerate(rows):
2020 new_row = Row()
2021 for (j,colname) in enumerate(colnames):
2022 value = row[j]
2023 tmp = tmps[j]
2024 if tmp:
2025 (tablename,fieldname,table,field,ft) = tmp
2026 if tablename in new_row:
2027 colset = new_row[tablename]
2028 else:
2029 colset = new_row[tablename] = Row()
2030 if tablename not in virtualtables:
2031 virtualtables.append(tablename)
2032 value = self.parse_value(value,ft,blob_decode)
2033 if field.filter_out:
2034 value = field.filter_out(value)
2035 colset[fieldname] = value
2036
2037
2038 if ft=='id' and fieldname!='id' and \
2039 not 'id' in table.fields:
2040 colset['id'] = value
2041
2042 if ft == 'id' and not cacheable:
2043
2044
2045
2046 if isinstance(self, GoogleDatastoreAdapter):
2047 id = value.key().id_or_name()
2048 colset[fieldname] = id
2049 colset.gae_item = value
2050 else:
2051 id = value
2052 colset.update_record = RecordUpdater(colset,table,id)
2053 colset.delete_record = RecordDeleter(table,id)
2054 for rfield in table._referenced_by:
2055 referee_link = db._referee_name and \
2056 db._referee_name % dict(
2057 table=rfield.tablename,field=rfield.name)
2058 if referee_link and not referee_link in colset:
2059 colset[referee_link] = LazySet(rfield,id)
2060 else:
2061 if not '_extra' in new_row:
2062 new_row['_extra'] = Row()
2063 new_row['_extra'][colname] = \
2064 self.parse_value(value,
2065 fields[j].type,blob_decode)
2066 new_column_name = \
2067 REGEX_SELECT_AS_PARSER.search(colname)
2068 if not new_column_name is None:
2069 column_name = new_column_name.groups(0)
2070 setattr(new_row,column_name[0],value)
2071 new_rows.append(new_row)
2072 rowsobj = Rows(db, new_rows, colnames, rawrows=rows)
2073
2074 for tablename in virtualtables:
2075
2076 table = db[tablename]
2077 fields_virtual = [(f,v) for (f,v) in table.iteritems()
2078 if isinstance(v,FieldVirtual)]
2079 fields_lazy = [(f,v) for (f,v) in table.iteritems()
2080 if isinstance(v,FieldMethod)]
2081 if fields_virtual or fields_lazy:
2082 for row in rowsobj.records:
2083 box = row[tablename]
2084 for f,v in fields_virtual:
2085 box[f] = v.f(row)
2086 for f,v in fields_lazy:
2087 box[f] = (v.handler or VirtualCommand)(v.f,row)
2088
2089
2090 for item in table.virtualfields:
2091 try:
2092 rowsobj = rowsobj.setvirtualfields(**{tablename:item})
2093 except (KeyError, AttributeError):
2094
2095 pass
2096 return rowsobj
2097
2099 tenant_fieldname = self.db._request_tenant
2100
2101 for tablename in tablenames:
2102 table = self.db[tablename]
2103
2104
2105 if table._common_filter != None:
2106 query = query & table._common_filter(query)
2107
2108
2109 if tenant_fieldname in table:
2110 default = table[tenant_fieldname].default
2111 if not default is None:
2112 newquery = table[tenant_fieldname] == default
2113 if query is None:
2114 query = newquery
2115 else:
2116 query = query & newquery
2117 return query
2118
2119 - def CASE(self,query,t,f):
2120 def represent(x):
2121 types = {type(True):'boolean',type(0):'integer',type(1.0):'double'}
2122 if x is None: return 'NULL'
2123 elif isinstance(x,Expression): return str(x)
2124 else: return self.represent(x,types.get(type(x),'string'))
2125 return Expression(self.db,'CASE WHEN %s THEN %s ELSE %s END' % \
2126 (self.expand(query),represent(t),represent(f)))
2127
2133 drivers = ('sqlite2','sqlite3')
2134
2135 can_select_for_update = None
2136
2138 return "web2py_extract('%s',%s)" % (what, self.expand(field))
2139
2140 @staticmethod
2142 table = {
2143 'year': (0, 4),
2144 'month': (5, 7),
2145 'day': (8, 10),
2146 'hour': (11, 13),
2147 'minute': (14, 16),
2148 'second': (17, 19),
2149 }
2150 try:
2151 if lookup != 'epoch':
2152 (i, j) = table[lookup]
2153 return int(s[i:j])
2154 else:
2155 return time.mktime(datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').timetuple())
2156 except:
2157 return None
2158
2159 @staticmethod
2161 return re.compile(expression).search(item) is not None
2162
2163 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2164 credential_decoder=IDENTITY, driver_args={},
2165 adapter_args={}, do_connect=True, after_connection=None):
2166 self.db = db
2167 self.dbengine = "sqlite"
2168 self.uri = uri
2169 if do_connect: self.find_driver(adapter_args)
2170 self.pool_size = 0
2171 self.folder = folder
2172 self.db_codec = db_codec
2173 self._after_connection = after_connection
2174 self.find_or_make_work_folder()
2175 path_encoding = sys.getfilesystemencoding() \
2176 or locale.getdefaultlocale()[1] or 'utf8'
2177 if uri.startswith('sqlite:memory'):
2178 dbpath = ':memory:'
2179 else:
2180 dbpath = uri.split('://',1)[1]
2181 if dbpath[0] != '/':
2182 if PYTHON_VERSION == 2:
2183 dbpath = pjoin(
2184 self.folder.decode(path_encoding).encode('utf8'), dbpath)
2185 else:
2186 dbpath = pjoin(self.folder, dbpath)
2187 if not 'check_same_thread' in driver_args:
2188 driver_args['check_same_thread'] = False
2189 if not 'detect_types' in driver_args and do_connect:
2190 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2191 def connector(dbpath=dbpath, driver_args=driver_args):
2192 return self.driver.Connection(dbpath, **driver_args)
2193 self.connector = connector
2194 if do_connect: self.reconnect()
2195
2201
2203 tablename = table._tablename
2204 return ['DELETE FROM %s;' % tablename,
2205 "DELETE FROM sqlite_sequence WHERE name='%s';" % tablename]
2206
2209
2210 - def REGEXP(self,first,second):
2211 return '(%s REGEXP %s)' % (self.expand(first),
2212 self.expand(second,'string'))
2213
2214 - def select(self, query, fields, attributes):
2215 """
2216 Simulate SELECT ... FOR UPDATE with BEGIN IMMEDIATE TRANSACTION.
2217 Note that the entire database, rather than one record, is locked
2218 (it will be locked eventually anyway by the following UPDATE).
2219 """
2220 if attributes.get('for_update', False) and not 'cache' in attributes:
2221 self.execute('BEGIN IMMEDIATE TRANSACTION;')
2222 return super(SQLiteAdapter, self).select(query, fields, attributes)
2223
2225 drivers = ('sqlite3','sqlite2')
2226
2227 types = copy.copy(BaseAdapter.types)
2228 types.update(geometry='GEOMETRY')
2229
2230 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2231 credential_decoder=IDENTITY, driver_args={},
2232 adapter_args={}, do_connect=True, srid=4326, after_connection=None):
2233 self.db = db
2234 self.dbengine = "spatialite"
2235 self.uri = uri
2236 if do_connect: self.find_driver(adapter_args)
2237 self.pool_size = 0
2238 self.folder = folder
2239 self.db_codec = db_codec
2240 self._after_connection = after_connection
2241 self.find_or_make_work_folder()
2242 self.srid = srid
2243 path_encoding = sys.getfilesystemencoding() \
2244 or locale.getdefaultlocale()[1] or 'utf8'
2245 if uri.startswith('spatialite:memory'):
2246 dbpath = ':memory:'
2247 else:
2248 dbpath = uri.split('://',1)[1]
2249 if dbpath[0] != '/':
2250 dbpath = pjoin(
2251 self.folder.decode(path_encoding).encode('utf8'), dbpath)
2252 if not 'check_same_thread' in driver_args:
2253 driver_args['check_same_thread'] = False
2254 if not 'detect_types' in driver_args and do_connect:
2255 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2256 def connector(dbpath=dbpath, driver_args=driver_args):
2257 return self.driver.Connection(dbpath, **driver_args)
2258 self.connector = connector
2259 if do_connect: self.reconnect()
2260
2273
2274
2275
2277 return 'AsGeoJSON(%s,%s,%s)' %(self.expand(first),
2278 second['precision'], second['options'])
2279
2280 - def ST_ASTEXT(self, first):
2281 return 'AsText(%s)' %(self.expand(first))
2282
2286
2290
2294
2298
2302
2304 return 'Simplify(%s,%s)' %(self.expand(first),
2305 self.expand(second, 'double'))
2306
2310
2314
2316 field_is_type = fieldtype.startswith
2317 if field_is_type('geo'):
2318 srid = 4326
2319 geotype, parms = fieldtype[:-1].split('(')
2320 parms = parms.split(',')
2321 if len(parms) >= 2:
2322 schema, srid = parms[:2]
2323
2324 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2325
2326
2327
2328
2329 return value
2330 return BaseAdapter.represent(self, obj, fieldtype)
2331
2334 drivers = ('zxJDBC_sqlite',)
2335
2336 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
2337 credential_decoder=IDENTITY, driver_args={},
2338 adapter_args={}, do_connect=True, after_connection=None):
2339 self.db = db
2340 self.dbengine = "sqlite"
2341 self.uri = uri
2342 if do_connect: self.find_driver(adapter_args)
2343 self.pool_size = pool_size
2344 self.folder = folder
2345 self.db_codec = db_codec
2346 self._after_connection = after_connection
2347 self.find_or_make_work_folder()
2348 path_encoding = sys.getfilesystemencoding() \
2349 or locale.getdefaultlocale()[1] or 'utf8'
2350 if uri.startswith('sqlite:memory'):
2351 dbpath = ':memory:'
2352 else:
2353 dbpath = uri.split('://',1)[1]
2354 if dbpath[0] != '/':
2355 dbpath = pjoin(
2356 self.folder.decode(path_encoding).encode('utf8'), dbpath)
2357 def connector(dbpath=dbpath,driver_args=driver_args):
2358 return self.driver.connect(
2359 self.driver.getConnection('jdbc:sqlite:'+dbpath),
2360 **driver_args)
2361 self.connector = connector
2362 if do_connect: self.reconnect()
2363
2368
2371
2374 drivers = ('MySQLdb','pymysql')
2375
2376 maxcharlength = 255
2377 commit_on_alter_table = True
2378 support_distributed_transaction = True
2379 types = {
2380 'boolean': 'CHAR(1)',
2381 'string': 'VARCHAR(%(length)s)',
2382 'text': 'LONGTEXT',
2383 'json': 'LONGTEXT',
2384 'password': 'VARCHAR(%(length)s)',
2385 'blob': 'LONGBLOB',
2386 'upload': 'VARCHAR(%(length)s)',
2387 'integer': 'INT',
2388 'bigint': 'BIGINT',
2389 'float': 'FLOAT',
2390 'double': 'DOUBLE',
2391 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2392 'date': 'DATE',
2393 'time': 'TIME',
2394 'datetime': 'DATETIME',
2395 'id': 'INT AUTO_INCREMENT NOT NULL',
2396 'reference': 'INT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2397 'list:integer': 'LONGTEXT',
2398 'list:string': 'LONGTEXT',
2399 'list:reference': 'LONGTEXT',
2400 'big-id': 'BIGINT AUTO_INCREMENT NOT NULL',
2401 'big-reference': 'BIGINT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2402 }
2403
2406
2409
2411 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field),
2412 parameters[0], parameters[1])
2413
2414 - def EPOCH(self, first):
2416
2417 - def REGEXP(self,first,second):
2418 return '(%s REGEXP %s)' % (self.expand(first),
2419 self.expand(second,'string'))
2420
2421 - def _drop(self,table,mode):
2422
2423 return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table,
2424 'SET FOREIGN_KEY_CHECKS=1;']
2425
2428
2432
2435
2438
2439 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
2440
2441 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2442 credential_decoder=IDENTITY, driver_args={},
2443 adapter_args={}, do_connect=True, after_connection=None):
2444 self.db = db
2445 self.dbengine = "mysql"
2446 self.uri = uri
2447 if do_connect: self.find_driver(adapter_args,uri)
2448 self.pool_size = pool_size
2449 self.folder = folder
2450 self.db_codec = db_codec
2451 self._after_connection = after_connection
2452 self.find_or_make_work_folder()
2453 ruri = uri.split('://',1)[1]
2454 m = self.REGEX_URI.match(ruri)
2455 if not m:
2456 raise SyntaxError(
2457 "Invalid URI string in DAL: %s" % self.uri)
2458 user = credential_decoder(m.group('user'))
2459 if not user:
2460 raise SyntaxError('User required')
2461 password = credential_decoder(m.group('password'))
2462 if not password:
2463 password = ''
2464 host = m.group('host')
2465 if not host:
2466 raise SyntaxError('Host name required')
2467 db = m.group('db')
2468 if not db:
2469 raise SyntaxError('Database name required')
2470 port = int(m.group('port') or '3306')
2471 charset = m.group('charset') or 'utf8'
2472 driver_args.update(db=db,
2473 user=credential_decoder(user),
2474 passwd=credential_decoder(password),
2475 host=host,
2476 port=port,
2477 charset=charset)
2478
2479
2480 def connector(driver_args=driver_args):
2481 return self.driver.connect(**driver_args)
2482 self.connector = connector
2483 if do_connect: self.reconnect()
2484
2486 self.execute('SET FOREIGN_KEY_CHECKS=1;')
2487 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
2488
2490 self.execute('select last_insert_id();')
2491 return int(self.cursor.fetchone()[0])
2492
2493 -class PostgreSQLAdapter(BaseAdapter):
2494 drivers = ('psycopg2','pg8000')
2495
2496 support_distributed_transaction = True
2497 types = {
2498 'boolean': 'CHAR(1)',
2499 'string': 'VARCHAR(%(length)s)',
2500 'text': 'TEXT',
2501 'json': 'TEXT',
2502 'password': 'VARCHAR(%(length)s)',
2503 'blob': 'BYTEA',
2504 'upload': 'VARCHAR(%(length)s)',
2505 'integer': 'INTEGER',
2506 'bigint': 'BIGINT',
2507 'float': 'FLOAT',
2508 'double': 'FLOAT8',
2509 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2510 'date': 'DATE',
2511 'time': 'TIME',
2512 'datetime': 'TIMESTAMP',
2513 'id': 'SERIAL PRIMARY KEY',
2514 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2515 'list:integer': 'TEXT',
2516 'list:string': 'TEXT',
2517 'list:reference': 'TEXT',
2518 'geometry': 'GEOMETRY',
2519 'geography': 'GEOGRAPHY',
2520 'big-id': 'BIGSERIAL PRIMARY KEY',
2521 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2522 }
2523
2524 - def varquote(self,name):
2525 return varquote_aux(name,'"%s"')
2526
2527 - def adapt(self,obj):
2528 if self.driver_name == 'psycopg2':
2529 return psycopg2_adapt(obj).getquoted()
2530 elif self.driver_name == 'pg8000':
2531 return "'%s'" % str(obj).replace("%","%%").replace("'","''")
2532 else:
2533 return "'%s'" % str(obj).replace("'","''")
2534
2535 - def sequence_name(self,table):
2536 return '%s_id_Seq' % table
2537
2540
2541 - def ADD(self, first, second):
2542 t = first.type
2543 if t in ('text','string','password', 'json', 'upload','blob'):
2544 return '(%s || %s)' % (self.expand(first), self.expand(second, t))
2545 else:
2546 return '(%s + %s)' % (self.expand(first), self.expand(second, t))
2547
2550
2551 - def prepare(self,key):
2552 self.execute("PREPARE TRANSACTION '%s';" % key)
2553
2554 - def commit_prepared(self,key):
2555 self.execute("COMMIT PREPARED '%s';" % key)
2556
2557 - def rollback_prepared(self,key):
2558 self.execute("ROLLBACK PREPARED '%s';" % key)
2559
2560 - def create_sequence_and_triggers(self, query, table, **args):
2561
2562
2563
2564
2565 self.execute(query)
2566
2567 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
2568
2569 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2570 credential_decoder=IDENTITY, driver_args={},
2571 adapter_args={}, do_connect=True, srid=4326,
2572 after_connection=None):
2573 self.db = db
2574 self.dbengine = "postgres"
2575 self.uri = uri
2576 if do_connect: self.find_driver(adapter_args,uri)
2577 self.pool_size = pool_size
2578 self.folder = folder
2579 self.db_codec = db_codec
2580 self._after_connection = after_connection
2581 self.srid = srid
2582 self.find_or_make_work_folder()
2583 ruri = uri.split('://',1)[1]
2584 m = self.REGEX_URI.match(ruri)
2585 if not m:
2586 raise SyntaxError("Invalid URI string in DAL")
2587 user = credential_decoder(m.group('user'))
2588 if not user:
2589 raise SyntaxError('User required')
2590 password = credential_decoder(m.group('password'))
2591 if not password:
2592 password = ''
2593 host = m.group('host')
2594 if not host:
2595 raise SyntaxError('Host name required')
2596 db = m.group('db')
2597 if not db:
2598 raise SyntaxError('Database name required')
2599 port = m.group('port') or '5432'
2600 sslmode = m.group('sslmode')
2601 if sslmode:
2602 msg = ("dbname='%s' user='%s' host='%s' "
2603 "port=%s password='%s' sslmode='%s'") \
2604 % (db, user, host, port, password, sslmode)
2605 else:
2606 msg = ("dbname='%s' user='%s' host='%s' "
2607 "port=%s password='%s'") \
2608 % (db, user, host, port, password)
2609
2610 self.__version__ = "%s %s" % (self.driver.__name__, self.driver.__version__)
2611 def connector(msg=msg,driver_args=driver_args):
2612 return self.driver.connect(msg,**driver_args)
2613 self.connector = connector
2614 if do_connect: self.reconnect()
2615
2616 - def after_connection(self):
2617 self.connection.set_client_encoding('UTF8')
2618 self.execute("SET standard_conforming_strings=on;")
2619 self.try_json()
2620
2621 - def lastrowid(self,table):
2622 self.execute("select currval('%s')" % table._sequence_name)
2623 return int(self.cursor.fetchone()[0])
2624
2625 - def try_json(self):
2626
2627
2628 if self.driver_name == "pg8000":
2629 supports_json = self.connection.server_version >= "9.2.0"
2630 elif (self.driver_name == "psycopg2") and \
2631 (self.driver.__version__ >= "2.0.12"):
2632 supports_json = self.connection.server_version >= 90200
2633 elif self.driver_name == "zxJDBC":
2634 supports_json = self.connection.dbversion >= "9.2.0"
2635 else: supports_json = None
2636 if supports_json: self.types["json"] = "JSON"
2637 else: LOGGER.debug("Your database version does not support the JSON data type (using TEXT instead)")
2638
2639 - def LIKE(self,first,second):
2640 args = (self.expand(first), self.expand(second,'string'))
2641 if not first.type in ('string', 'text', 'json'):
2642 return '(CAST(%s AS CHAR(%s)) LIKE %s)' % (args[0], first.length, args[1])
2643 else:
2644 return '(%s LIKE %s)' % args
2645
2646 - def ILIKE(self,first,second):
2647 args = (self.expand(first), self.expand(second,'string'))
2648 if not first.type in ('string', 'text', 'json'):
2649 return '(CAST(%s AS CHAR(%s)) LIKE %s)' % (args[0], first.length, args[1])
2650 else:
2651 return '(%s ILIKE %s)' % args
2652
2653 - def REGEXP(self,first,second):
2654 return '(%s ~ %s)' % (self.expand(first),
2655 self.expand(second,'string'))
2656
2657 - def STARTSWITH(self,first,second):
2658 return '(%s ILIKE %s)' % (self.expand(first),
2659 self.expand(second+'%','string'))
2660
2661 - def ENDSWITH(self,first,second):
2662 return '(%s ILIKE %s)' % (self.expand(first),
2663 self.expand('%'+second,'string'))
2664
2665 - def CONTAINS(self,first,second,case_sensitive=False):
2666 if first.type in ('string','text', 'json'):
2667 key = '%'+str(second).replace('%','%%')+'%'
2668 elif first.type.startswith('list:'):
2669 key = '%|'+str(second).replace('|','||').replace('%','%%')+'|%'
2670 op = case_sensitive and self.LIKE or self.ILIKE
2671 return op(first,key)
2672
2673
2674
2675 - def ST_ASGEOJSON(self, first, second):
2676 """
2677 http://postgis.org/docs/ST_AsGeoJSON.html
2678 """
2679 return 'ST_AsGeoJSON(%s,%s,%s,%s)' %(second['version'],
2680 self.expand(first), second['precision'], second['options'])
2681
2682 - def ST_ASTEXT(self, first):
2683 """
2684 http://postgis.org/docs/ST_AsText.html
2685 """
2686 return 'ST_AsText(%s)' %(self.expand(first))
2687
2688 - def ST_X(self, first):
2689 """
2690 http://postgis.org/docs/ST_X.html
2691 """
2692 return 'ST_X(%s)' %(self.expand(first))
2693
2694 - def ST_Y(self, first):
2695 """
2696 http://postgis.org/docs/ST_Y.html
2697 """
2698 return 'ST_Y(%s)' %(self.expand(first))
2699
2700 - def ST_CONTAINS(self, first, second):
2701 """
2702 http://postgis.org/docs/ST_Contains.html
2703 """
2704 return 'ST_Contains(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2705
2706 - def ST_DISTANCE(self, first, second):
2707 """
2708 http://postgis.org/docs/ST_Distance.html
2709 """
2710 return 'ST_Distance(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2711
2712 - def ST_EQUALS(self, first, second):
2713 """
2714 http://postgis.org/docs/ST_Equals.html
2715 """
2716 return 'ST_Equals(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2717
2718 - def ST_INTERSECTS(self, first, second):
2719 """
2720 http://postgis.org/docs/ST_Intersects.html
2721 """
2722 return 'ST_Intersects(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2723
2724 - def ST_OVERLAPS(self, first, second):
2725 """
2726 http://postgis.org/docs/ST_Overlaps.html
2727 """
2728 return 'ST_Overlaps(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2729
2730 - def ST_SIMPLIFY(self, first, second):
2731 """
2732 http://postgis.org/docs/ST_Simplify.html
2733 """
2734 return 'ST_Simplify(%s,%s)' %(self.expand(first), self.expand(second, 'double'))
2735
2736 - def ST_TOUCHES(self, first, second):
2737 """
2738 http://postgis.org/docs/ST_Touches.html
2739 """
2740 return 'ST_Touches(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2741
2742 - def ST_WITHIN(self, first, second):
2743 """
2744 http://postgis.org/docs/ST_Within.html
2745 """
2746 return 'ST_Within(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2747
2748 - def represent(self, obj, fieldtype):
2749 field_is_type = fieldtype.startswith
2750 if field_is_type('geo'):
2751 srid = 4326
2752 geotype, parms = fieldtype[:-1].split('(')
2753 parms = parms.split(',')
2754 if len(parms) >= 2:
2755 schema, srid = parms[:2]
2756 if field_is_type('geometry'):
2757 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2758 elif field_is_type('geography'):
2759 value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj)
2760
2761
2762 return value
2763 return BaseAdapter.represent(self, obj, fieldtype)
2764
2765 -class NewPostgreSQLAdapter(PostgreSQLAdapter):
2766 drivers = ('psycopg2','pg8000')
2767
2768 types = {
2769 'boolean': 'CHAR(1)',
2770 'string': 'VARCHAR(%(length)s)',
2771 'text': 'TEXT',
2772 'json': 'TEXT',
2773 'password': 'VARCHAR(%(length)s)',
2774 'blob': 'BYTEA',
2775 'upload': 'VARCHAR(%(length)s)',
2776 'integer': 'INTEGER',
2777 'bigint': 'BIGINT',
2778 'float': 'FLOAT',
2779 'double': 'FLOAT8',
2780 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2781 'date': 'DATE',
2782 'time': 'TIME',
2783 'datetime': 'TIMESTAMP',
2784 'id': 'SERIAL PRIMARY KEY',
2785 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2786 'list:integer': 'BIGINT[]',
2787 'list:string': 'TEXT[]',
2788 'list:reference': 'BIGINT[]',
2789 'geometry': 'GEOMETRY',
2790 'geography': 'GEOGRAPHY',
2791 'big-id': 'BIGSERIAL PRIMARY KEY',
2792 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2793 }
2794
2795 - def parse_list_integers(self, value, field_type):
2797
2798 - def parse_list_references(self, value, field_type):
2799 return [self.parse_reference(r, field_type[5:]) for r in value]
2800
2801 - def parse_list_strings(self, value, field_type):
2803
2804 - def represent(self, obj, fieldtype):
2805 field_is_type = fieldtype.startswith
2806 if field_is_type('list:'):
2807 if not obj:
2808 obj = []
2809 elif not isinstance(obj, (list, tuple)):
2810 obj = [obj]
2811 if field_is_type('list:string'):
2812 obj = map(str,obj)
2813 else:
2814 obj = map(int,obj)
2815 return 'ARRAY[%s]' % ','.join(repr(item) for item in obj)
2816 return BaseAdapter.represent(self, obj, fieldtype)
2817
2818
2819 -class JDBCPostgreSQLAdapter(PostgreSQLAdapter):
2820 drivers = ('zxJDBC',)
2821
2822 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$')
2823
2824 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2825 credential_decoder=IDENTITY, driver_args={},
2826 adapter_args={}, do_connect=True, after_connection=None ):
2827 self.db = db
2828 self.dbengine = "postgres"
2829 self.uri = uri
2830 if do_connect: self.find_driver(adapter_args,uri)
2831 self.pool_size = pool_size
2832 self.folder = folder
2833 self.db_codec = db_codec
2834 self._after_connection = after_connection
2835 self.find_or_make_work_folder()
2836 ruri = uri.split('://',1)[1]
2837 m = self.REGEX_URI.match(ruri)
2838 if not m:
2839 raise SyntaxError("Invalid URI string in DAL")
2840 user = credential_decoder(m.group('user'))
2841 if not user:
2842 raise SyntaxError('User required')
2843 password = credential_decoder(m.group('password'))
2844 if not password:
2845 password = ''
2846 host = m.group('host')
2847 if not host:
2848 raise SyntaxError('Host name required')
2849 db = m.group('db')
2850 if not db:
2851 raise SyntaxError('Database name required')
2852 port = m.group('port') or '5432'
2853 msg = ('jdbc:postgresql://%s:%s/%s' % (host, port, db), user, password)
2854 def connector(msg=msg,driver_args=driver_args):
2855 return self.driver.connect(*msg,**driver_args)
2856 self.connector = connector
2857 if do_connect: self.reconnect()
2858
2859 - def after_connection(self):
2860 self.connection.set_client_encoding('UTF8')
2861 self.execute('BEGIN;')
2862 self.execute("SET CLIENT_ENCODING TO 'UNICODE';")
2863 self.try_json()
2864
2867 drivers = ('cx_Oracle',)
2868
2869 commit_on_alter_table = False
2870 types = {
2871 'boolean': 'CHAR(1)',
2872 'string': 'VARCHAR2(%(length)s)',
2873 'text': 'CLOB',
2874 'json': 'CLOB',
2875 'password': 'VARCHAR2(%(length)s)',
2876 'blob': 'CLOB',
2877 'upload': 'VARCHAR2(%(length)s)',
2878 'integer': 'INT',
2879 'bigint': 'NUMBER',
2880 'float': 'FLOAT',
2881 'double': 'BINARY_DOUBLE',
2882 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2883 'date': 'DATE',
2884 'time': 'CHAR(8)',
2885 'datetime': 'DATE',
2886 'id': 'NUMBER PRIMARY KEY',
2887 'reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2888 'list:integer': 'CLOB',
2889 'list:string': 'CLOB',
2890 'list:reference': 'CLOB',
2891 'big-id': 'NUMBER PRIMARY KEY',
2892 'big-reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2893 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2894 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
2895 }
2896
2898 return '%s_sequence' % tablename
2899
2901 return '%s_trigger' % tablename
2902
2904 return 'LEFT OUTER JOIN'
2905
2907 return 'dbms_random.value'
2908
2909 - def NOT_NULL(self,default,field_type):
2910 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
2911
2912 - def _drop(self,table,mode):
2915
2916 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
2917 if limitby:
2918 (lmin, lmax) = limitby
2919 if len(sql_w) > 1:
2920 sql_w_row = sql_w + ' AND w_row > %i' % lmin
2921 else:
2922 sql_w_row = 'WHERE w_row > %i' % lmin
2923 return 'SELECT %s %s FROM (SELECT w_tmp.*, ROWNUM w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNUM<=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
2924 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
2925
2931
2933 if fieldtype == 'blob':
2934 obj = base64.b64encode(str(obj))
2935 return ":CLOB('%s')" % obj
2936 elif fieldtype == 'date':
2937 if isinstance(obj, (datetime.date, datetime.datetime)):
2938 obj = obj.isoformat()[:10]
2939 else:
2940 obj = str(obj)
2941 return "to_date('%s','yyyy-mm-dd')" % obj
2942 elif fieldtype == 'datetime':
2943 if isinstance(obj, datetime.datetime):
2944 obj = obj.isoformat()[:19].replace('T',' ')
2945 elif isinstance(obj, datetime.date):
2946 obj = obj.isoformat()[:10]+' 00:00:00'
2947 else:
2948 obj = str(obj)
2949 return "to_date('%s','yyyy-mm-dd hh24:mi:ss')" % obj
2950 return None
2951
2952 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2953 credential_decoder=IDENTITY, driver_args={},
2954 adapter_args={}, do_connect=True, after_connection=None):
2955 self.db = db
2956 self.dbengine = "oracle"
2957 self.uri = uri
2958 if do_connect: self.find_driver(adapter_args,uri)
2959 self.pool_size = pool_size
2960 self.folder = folder
2961 self.db_codec = db_codec
2962 self._after_connection = after_connection
2963 self.find_or_make_work_folder()
2964 ruri = uri.split('://',1)[1]
2965 if not 'threaded' in driver_args:
2966 driver_args['threaded']=True
2967 def connector(uri=ruri,driver_args=driver_args):
2968 return self.driver.connect(uri,**driver_args)
2969 self.connector = connector
2970 if do_connect: self.reconnect()
2971
2973 self.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
2974 self.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
2975
2976 oracle_fix = re.compile("[^']*('[^']*'[^']*)*\:(?P<clob>CLOB\('([^']+|'')*'\))")
2977
2978 - def execute(self, command, args=None):
2979 args = args or []
2980 i = 1
2981 while True:
2982 m = self.oracle_fix.match(command)
2983 if not m:
2984 break
2985 command = command[:m.start('clob')] + str(i) + command[m.end('clob'):]
2986 args.append(m.group('clob')[6:-2].replace("''", "'"))
2987 i += 1
2988 if command[-1:]==';':
2989 command = command[:-1]
2990 return self.log_execute(command, args)
2991
2993 tablename = table._tablename
2994 sequence_name = table._sequence_name
2995 trigger_name = table._trigger_name
2996 self.execute(query)
2997 self.execute('CREATE SEQUENCE %s START WITH 1 INCREMENT BY 1 NOMAXVALUE MINVALUE -1;' % sequence_name)
2998 self.execute("""
2999 CREATE OR REPLACE TRIGGER %(trigger_name)s BEFORE INSERT ON %(tablename)s FOR EACH ROW
3000 DECLARE
3001 curr_val NUMBER;
3002 diff_val NUMBER;
3003 PRAGMA autonomous_transaction;
3004 BEGIN
3005 IF :NEW.id IS NOT NULL THEN
3006 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
3007 diff_val := :NEW.id - curr_val - 1;
3008 IF diff_val != 0 THEN
3009 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by '|| diff_val;
3010 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
3011 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by 1';
3012 END IF;
3013 END IF;
3014 SELECT %(sequence_name)s.nextval INTO :NEW.id FROM DUAL;
3015 END;
3016 """ % dict(trigger_name=trigger_name, tablename=tablename, sequence_name=sequence_name))
3017
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3033 if any(x[1]==cx_Oracle.CLOB for x in self.cursor.description):
3034 return [tuple([(c.read() if type(c) == cx_Oracle.LOB else c) \
3035 for c in r]) for r in self.cursor]
3036 else:
3037 return self.cursor.fetchall()
3038
3040 drivers = ('pyodbc',)
3041
3042 types = {
3043 'boolean': 'BIT',
3044 'string': 'VARCHAR(%(length)s)',
3045 'text': 'TEXT',
3046 'json': 'TEXT',
3047 'password': 'VARCHAR(%(length)s)',
3048 'blob': 'IMAGE',
3049 'upload': 'VARCHAR(%(length)s)',
3050 'integer': 'INT',
3051 'bigint': 'BIGINT',
3052 'float': 'FLOAT',
3053 'double': 'FLOAT',
3054 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3055 'date': 'DATETIME',
3056 'time': 'CHAR(8)',
3057 'datetime': 'DATETIME',
3058 'id': 'INT IDENTITY PRIMARY KEY',
3059 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3060 'list:integer': 'TEXT',
3061 'list:string': 'TEXT',
3062 'list:reference': 'TEXT',
3063 'geometry': 'geometry',
3064 'geography': 'geography',
3065 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3066 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3067 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3068 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3069 }
3070
3072 return '; ALTER TABLE %s ADD ' % tablename
3073
3076
3078 return "DATEPART(%s,%s)" % (what, self.expand(field))
3079
3081 return 'LEFT OUTER JOIN'
3082
3085
3088
3090 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
3091
3093 return 'PRIMARY KEY CLUSTERED (%s)' % key
3094
3096 if what == 'LENGTH':
3097 what = 'LEN'
3098 return "%s(%s)" % (what, self.expand(first))
3099
3100
3101 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3102 if limitby:
3103 (lmin, lmax) = limitby
3104 sql_s += ' TOP %i' % lmax
3105 if 'GROUP BY' in sql_o:
3106 sql_o = sql_o[:sql_o.find('ORDER BY ')]
3107 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3108
3109 TRUE = 1
3110 FALSE = 0
3111
3112 REGEX_DSN = re.compile('^(?P<dsn>.+)$')
3113 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?(?P<urlargs>.*))?$')
3114 REGEX_ARGPATTERN = re.compile('(?P<argkey>[^=]+)=(?P<argvalue>[^&]*)')
3115
3116 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3117 credential_decoder=IDENTITY, driver_args={},
3118 adapter_args={}, do_connect=True, srid=4326,
3119 after_connection=None):
3120 self.db = db
3121 self.dbengine = "mssql"
3122 self.uri = uri
3123 if do_connect: self.find_driver(adapter_args,uri)
3124 self.pool_size = pool_size
3125 self.folder = folder
3126 self.db_codec = db_codec
3127 self._after_connection = after_connection
3128 self.srid = srid
3129 self.find_or_make_work_folder()
3130
3131 ruri = uri.split('://',1)[1]
3132 if '@' not in ruri:
3133 try:
3134 m = self.REGEX_DSN.match(ruri)
3135 if not m:
3136 raise SyntaxError(
3137 'Parsing uri string(%s) has no result' % self.uri)
3138 dsn = m.group('dsn')
3139 if not dsn:
3140 raise SyntaxError('DSN required')
3141 except SyntaxError:
3142 e = sys.exc_info()[1]
3143 LOGGER.error('NdGpatch error')
3144 raise e
3145
3146 cnxn = dsn
3147 else:
3148 m = self.REGEX_URI.match(ruri)
3149 if not m:
3150 raise SyntaxError(
3151 "Invalid URI string in DAL: %s" % self.uri)
3152 user = credential_decoder(m.group('user'))
3153 if not user:
3154 raise SyntaxError('User required')
3155 password = credential_decoder(m.group('password'))
3156 if not password:
3157 password = ''
3158 host = m.group('host')
3159 if not host:
3160 raise SyntaxError('Host name required')
3161 db = m.group('db')
3162 if not db:
3163 raise SyntaxError('Database name required')
3164 port = m.group('port') or '1433'
3165
3166
3167
3168 argsdict = { 'DRIVER':'{SQL Server}' }
3169 urlargs = m.group('urlargs') or ''
3170 for argmatch in self.REGEX_ARGPATTERN.finditer(urlargs):
3171 argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue')
3172 urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()])
3173 cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \
3174 % (host, port, db, user, password, urlargs)
3175 def connector(cnxn=cnxn,driver_args=driver_args):
3176 return self.driver.connect(cnxn,**driver_args)
3177 self.connector = connector
3178 if do_connect: self.reconnect()
3179
3181
3182 self.execute('SELECT SCOPE_IDENTITY();')
3183 return int(self.cursor.fetchone()[0])
3184
3186 return pyodbc.IntegrityError
3187
3188 - def rowslice(self,rows,minimum=0,maximum=None):
3189 if maximum is None:
3190 return rows[minimum:]
3191 return rows[minimum:maximum]
3192
3193 - def EPOCH(self, first):
3194 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3195
3196
3197
3198
3199
3200 - def ST_ASTEXT(self, first):
3201 return '%s.STAsText()' %(self.expand(first))
3202
3205
3208
3211
3214
3217
3218
3219
3222
3225
3227 field_is_type = fieldtype.startswith
3228 if field_is_type('geometry'):
3229 srid = 0
3230 geotype, parms = fieldtype[:-1].split('(')
3231 if parms:
3232 srid = parms
3233 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3234 elif fieldtype == 'geography':
3235 srid = 4326
3236 geotype, parms = fieldtype[:-1].split('(')
3237 if parms:
3238 srid = parms
3239 return "geography::STGeomFromText('%s',%s)" %(obj, srid)
3240
3241
3242 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3243 return BaseAdapter.represent(self, obj, fieldtype)
3244
3247 """ experimental support for pagination in MSSQL"""
3248 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3249 if limitby:
3250 (lmin, lmax) = limitby
3251 if lmin == 0:
3252 sql_s += ' TOP %i' % lmax
3253 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3254 lmin += 1
3255 sql_o_inner = sql_o[sql_o.find('ORDER BY ')+9:]
3256 sql_g_inner = sql_o[:sql_o.find('ORDER BY ')]
3257 sql_f_outer = ['f_%s' % f for f in range(len(sql_f.split(',')))]
3258 sql_f_inner = [f for f in sql_f.split(',')]
3259 sql_f_iproxy = ['%s AS %s' % (o, n) for (o, n) in zip(sql_f_inner, sql_f_outer)]
3260 sql_f_iproxy = ', '.join(sql_f_iproxy)
3261 sql_f_oproxy = ', '.join(sql_f_outer)
3262 return 'SELECT %s %s FROM (SELECT %s ROW_NUMBER() OVER (ORDER BY %s) AS w_row, %s FROM %s%s%s) TMP WHERE w_row BETWEEN %i AND %s;' % (sql_s,sql_f_oproxy,sql_s,sql_f,sql_f_iproxy,sql_t,sql_w,sql_g_inner,lmin,lmax)
3263 return 'SELECT %s %s FROM %s%s%s;' % (sql_s,sql_f,sql_t,sql_w,sql_o)
3264 - def rowslice(self,rows,minimum=0,maximum=None):
3266
3269 drivers = ('pyodbc',)
3270
3271 types = {
3272 'boolean': 'CHAR(1)',
3273 'string': 'NVARCHAR(%(length)s)',
3274 'text': 'NTEXT',
3275 'json': 'NTEXT',
3276 'password': 'NVARCHAR(%(length)s)',
3277 'blob': 'IMAGE',
3278 'upload': 'NVARCHAR(%(length)s)',
3279 'integer': 'INT',
3280 'bigint': 'BIGINT',
3281 'float': 'FLOAT',
3282 'double': 'FLOAT',
3283 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3284 'date': 'DATETIME',
3285 'time': 'CHAR(8)',
3286 'datetime': 'DATETIME',
3287 'id': 'INT IDENTITY PRIMARY KEY',
3288 'reference': 'INT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3289 'list:integer': 'NTEXT',
3290 'list:string': 'NTEXT',
3291 'list:reference': 'NTEXT',
3292 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3293 'big-reference': 'BIGINT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3294 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3295 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3296 }
3297
3299 value = BaseAdapter.represent(self, obj, fieldtype)
3300 if fieldtype in ('string','text', 'json') and value[:1]=="'":
3301 value = 'N'+value
3302 return value
3303
3306
3309 drivers = ('Sybase',)
3310
3311 types = {
3312 'boolean': 'BIT',
3313 'string': 'CHAR VARYING(%(length)s)',
3314 'text': 'TEXT',
3315 'json': 'TEXT',
3316 'password': 'CHAR VARYING(%(length)s)',
3317 'blob': 'IMAGE',
3318 'upload': 'CHAR VARYING(%(length)s)',
3319 'integer': 'INT',
3320 'bigint': 'BIGINT',
3321 'float': 'FLOAT',
3322 'double': 'FLOAT',
3323 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3324 'date': 'DATETIME',
3325 'time': 'CHAR(8)',
3326 'datetime': 'DATETIME',
3327 'id': 'INT IDENTITY PRIMARY KEY',
3328 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3329 'list:integer': 'TEXT',
3330 'list:string': 'TEXT',
3331 'list:reference': 'TEXT',
3332 'geometry': 'geometry',
3333 'geography': 'geography',
3334 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3335 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3336 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3337 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3338 }
3339
3340
3341 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3342 credential_decoder=IDENTITY, driver_args={},
3343 adapter_args={}, do_connect=True, srid=4326,
3344 after_connection=None):
3345 self.db = db
3346 self.dbengine = "sybase"
3347 self.uri = uri
3348 if do_connect: self.find_driver(adapter_args,uri)
3349 self.pool_size = pool_size
3350 self.folder = folder
3351 self.db_codec = db_codec
3352 self._after_connection = after_connection
3353 self.srid = srid
3354 self.find_or_make_work_folder()
3355
3356 ruri = uri.split('://',1)[1]
3357 if '@' not in ruri:
3358 try:
3359 m = self.REGEX_DSN.match(ruri)
3360 if not m:
3361 raise SyntaxError(
3362 'Parsing uri string(%s) has no result' % self.uri)
3363 dsn = m.group('dsn')
3364 if not dsn:
3365 raise SyntaxError('DSN required')
3366 except SyntaxError:
3367 e = sys.exc_info()[1]
3368 LOGGER.error('NdGpatch error')
3369 raise e
3370 else:
3371 m = self.REGEX_URI.match(uri)
3372 if not m:
3373 raise SyntaxError(
3374 "Invalid URI string in DAL: %s" % self.uri)
3375 user = credential_decoder(m.group('user'))
3376 if not user:
3377 raise SyntaxError('User required')
3378 password = credential_decoder(m.group('password'))
3379 if not password:
3380 password = ''
3381 host = m.group('host')
3382 if not host:
3383 raise SyntaxError('Host name required')
3384 db = m.group('db')
3385 if not db:
3386 raise SyntaxError('Database name required')
3387 port = m.group('port') or '1433'
3388
3389 dsn = 'sybase:host=%s:%s;dbname=%s' % (host,port,db)
3390
3391 driver_args.update(user = credential_decoder(user),
3392 password = credential_decoder(password))
3393
3394 def connector(dsn=dsn,driver_args=driver_args):
3395 return self.driver.connect(dsn,**driver_args)
3396 self.connector = connector
3397 if do_connect: self.reconnect()
3398
3401
3404 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3405
3406 commit_on_alter_table = False
3407 support_distributed_transaction = True
3408 types = {
3409 'boolean': 'CHAR(1)',
3410 'string': 'VARCHAR(%(length)s)',
3411 'text': 'BLOB SUB_TYPE 1',
3412 'json': 'BLOB SUB_TYPE 1',
3413 'password': 'VARCHAR(%(length)s)',
3414 'blob': 'BLOB SUB_TYPE 0',
3415 'upload': 'VARCHAR(%(length)s)',
3416 'integer': 'INTEGER',
3417 'bigint': 'BIGINT',
3418 'float': 'FLOAT',
3419 'double': 'DOUBLE PRECISION',
3420 'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
3421 'date': 'DATE',
3422 'time': 'TIME',
3423 'datetime': 'TIMESTAMP',
3424 'id': 'INTEGER PRIMARY KEY',
3425 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3426 'list:integer': 'BLOB SUB_TYPE 1',
3427 'list:string': 'BLOB SUB_TYPE 1',
3428 'list:reference': 'BLOB SUB_TYPE 1',
3429 'big-id': 'BIGINT PRIMARY KEY',
3430 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3431 }
3432
3434 return 'genid_%s' % tablename
3435
3437 return 'trg_id_%s' % tablename
3438
3441
3442 - def EPOCH(self, first):
3443 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3444
3445 - def NOT_NULL(self,default,field_type):
3446 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
3447
3449 return 'SUBSTRING(%s from %s for %s)' % (self.expand(field), parameters[0], parameters[1])
3450
3452 "case in-sensitive like operator"
3453 return '(%s CONTAINING %s)' % (self.expand(first),
3454 self.expand(second, 'string'))
3455
3456 - def CONTAINS(self, first, second, case_sensitive=False):
3462
3463 - def _drop(self,table,mode):
3466
3467 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3468 if limitby:
3469 (lmin, lmax) = limitby
3470 sql_s = ' FIRST %i SKIP %i %s' % (lmax - lmin, lmin, sql_s)
3471 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3472
3474 return ['DELETE FROM %s;' % table._tablename,
3475 'SET GENERATOR %s TO 0;' % table._sequence_name]
3476
3477 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+?)(\?set_encoding=(?P<charset>\w+))?$')
3478
3479 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3480 credential_decoder=IDENTITY, driver_args={},
3481 adapter_args={}, do_connect=True, after_connection=None):
3482 self.db = db
3483 self.dbengine = "firebird"
3484 self.uri = uri
3485 if do_connect: self.find_driver(adapter_args,uri)
3486 self.pool_size = pool_size
3487 self.folder = folder
3488 self.db_codec = db_codec
3489 self._after_connection = after_connection
3490 self.find_or_make_work_folder()
3491 ruri = uri.split('://',1)[1]
3492 m = self.REGEX_URI.match(ruri)
3493 if not m:
3494 raise SyntaxError("Invalid URI string in DAL: %s" % self.uri)
3495 user = credential_decoder(m.group('user'))
3496 if not user:
3497 raise SyntaxError('User required')
3498 password = credential_decoder(m.group('password'))
3499 if not password:
3500 password = ''
3501 host = m.group('host')
3502 if not host:
3503 raise SyntaxError('Host name required')
3504 port = int(m.group('port') or 3050)
3505 db = m.group('db')
3506 if not db:
3507 raise SyntaxError('Database name required')
3508 charset = m.group('charset') or 'UTF8'
3509 driver_args.update(dsn='%s/%s:%s' % (host,port,db),
3510 user = credential_decoder(user),
3511 password = credential_decoder(password),
3512 charset = charset)
3513
3514 def connector(driver_args=driver_args):
3515 return self.driver.connect(**driver_args)
3516 self.connector = connector
3517 if do_connect: self.reconnect()
3518
3527
3532
3535 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3536
3537 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<path>[^\?]+)(\?set_encoding=(?P<charset>\w+))?$')
3538
3539 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3540 credential_decoder=IDENTITY, driver_args={},
3541 adapter_args={}, do_connect=True, after_connection=None):
3542 self.db = db
3543 self.dbengine = "firebird"
3544 self.uri = uri
3545 if do_connect: self.find_driver(adapter_args,uri)
3546 self.pool_size = pool_size
3547 self.folder = folder
3548 self.db_codec = db_codec
3549 self._after_connection = after_connection
3550 self.find_or_make_work_folder()
3551 ruri = uri.split('://',1)[1]
3552 m = self.REGEX_URI.match(ruri)
3553 if not m:
3554 raise SyntaxError(
3555 "Invalid URI string in DAL: %s" % self.uri)
3556 user = credential_decoder(m.group('user'))
3557 if not user:
3558 raise SyntaxError('User required')
3559 password = credential_decoder(m.group('password'))
3560 if not password:
3561 password = ''
3562 pathdb = m.group('path')
3563 if not pathdb:
3564 raise SyntaxError('Path required')
3565 charset = m.group('charset')
3566 if not charset:
3567 charset = 'UTF8'
3568 host = ''
3569 driver_args.update(host=host,
3570 database=pathdb,
3571 user=credential_decoder(user),
3572 password=credential_decoder(password),
3573 charset=charset)
3574
3575 def connector(driver_args=driver_args):
3576 return self.driver.connect(**driver_args)
3577 self.connector = connector
3578 if do_connect: self.reconnect()
3579
3685
3690
3693
3696
3708
3710 drivers = ('pyodbc',)
3711
3712 types = {
3713 'boolean': 'CHAR(1)',
3714 'string': 'VARCHAR(%(length)s)',
3715 'text': 'CLOB',
3716 'json': 'CLOB',
3717 'password': 'VARCHAR(%(length)s)',
3718 'blob': 'BLOB',
3719 'upload': 'VARCHAR(%(length)s)',
3720 'integer': 'INT',
3721 'bigint': 'BIGINT',
3722 'float': 'REAL',
3723 'double': 'DOUBLE',
3724 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3725 'date': 'DATE',
3726 'time': 'TIME',
3727 'datetime': 'TIMESTAMP',
3728 'id': 'INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3729 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3730 'list:integer': 'CLOB',
3731 'list:string': 'CLOB',
3732 'list:reference': 'CLOB',
3733 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3734 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3735 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3736 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3737 }
3738
3740 return 'LEFT OUTER JOIN'
3741
3744
3745 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3746 if limitby:
3747 (lmin, lmax) = limitby
3748 sql_o += ' FETCH FIRST %i ROWS ONLY' % lmax
3749 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3750
3752 if fieldtype == 'blob':
3753 obj = base64.b64encode(str(obj))
3754 return "BLOB('%s')" % obj
3755 elif fieldtype == 'datetime':
3756 if isinstance(obj, datetime.datetime):
3757 obj = obj.isoformat()[:19].replace('T','-').replace(':','.')
3758 elif isinstance(obj, datetime.date):
3759 obj = obj.isoformat()[:10]+'-00.00.00'
3760 return "'%s'" % obj
3761 return None
3762
3763 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3764 credential_decoder=IDENTITY, driver_args={},
3765 adapter_args={}, do_connect=True, after_connection=None):
3766 self.db = db
3767 self.dbengine = "db2"
3768 self.uri = uri
3769 if do_connect: self.find_driver(adapter_args,uri)
3770 self.pool_size = pool_size
3771 self.folder = folder
3772 self.db_codec = db_codec
3773 self._after_connection = after_connection
3774 self.find_or_make_work_folder()
3775 ruri = uri.split('://', 1)[1]
3776 def connector(cnxn=ruri,driver_args=driver_args):
3777 return self.driver.connect(cnxn,**driver_args)
3778 self.connector = connector
3779 if do_connect: self.reconnect()
3780
3782 if command[-1:]==';':
3783 command = command[:-1]
3784 return self.log_execute(command)
3785
3787 self.execute('SELECT DISTINCT IDENTITY_VAL_LOCAL() FROM %s;' % table)
3788 return int(self.cursor.fetchone()[0])
3789
3790 - def rowslice(self,rows,minimum=0,maximum=None):
3791 if maximum is None:
3792 return rows[minimum:]
3793 return rows[minimum:maximum]
3794
3797 drivers = ('pyodbc',)
3798
3799 types = {
3800 'boolean': 'CHAR(1)',
3801 'string': 'VARCHAR(%(length)s)',
3802 'text': 'CLOB',
3803 'json': 'CLOB',
3804 'password': 'VARCHAR(%(length)s)',
3805 'blob': 'BLOB',
3806 'upload': 'VARCHAR(%(length)s)',
3807 'integer': 'INT',
3808 'bigint': 'BIGINT',
3809 'float': 'REAL',
3810 'double': 'DOUBLE',
3811 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3812 'date': 'DATE',
3813 'time': 'TIME',
3814 'datetime': 'TIMESTAMP',
3815
3816
3817 'id': 'INT GENERATED ALWAYS AS IDENTITY',
3818 'reference': 'INT',
3819 'list:integer': 'CLOB',
3820 'list:string': 'CLOB',
3821 'list:reference': 'CLOB',
3822 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY',
3823 'big-reference': 'BIGINT',
3824 'reference FK': ' REFERENCES %(foreign_key)s',
3825 'reference TFK': ' FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s)',
3826 }
3827
3828 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3829 credential_decoder=IDENTITY, driver_args={},
3830 adapter_args={}, do_connect=True, after_connection=None):
3831 self.db = db
3832 self.dbengine = "teradata"
3833 self.uri = uri
3834 if do_connect: self.find_driver(adapter_args,uri)
3835 self.pool_size = pool_size
3836 self.folder = folder
3837 self.db_codec = db_codec
3838 self._after_connection = after_connection
3839 self.find_or_make_work_folder()
3840 ruri = uri.split('://', 1)[1]
3841 def connector(cnxn=ruri,driver_args=driver_args):
3842 return self.driver.connect(cnxn,**driver_args)
3843 self.connector = connector
3844 if do_connect: self.reconnect()
3845
3847 return 'LEFT OUTER JOIN'
3848
3849
3850 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3851 if limitby:
3852 (lmin, lmax) = limitby
3853 sql_s += ' TOP %i' % lmax
3854 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3855
3857 tablename = table._tablename
3858 return ['DELETE FROM %s ALL;' % (tablename)]
3859
3860 INGRES_SEQNAME='ii***lineitemsequence'
3865 drivers = ('ingresdbi',)
3866
3867 types = {
3868 'boolean': 'CHAR(1)',
3869 'string': 'VARCHAR(%(length)s)',
3870 'text': 'CLOB',
3871 'json': 'CLOB',
3872 'password': 'VARCHAR(%(length)s)',
3873 'blob': 'BLOB',
3874 'upload': 'VARCHAR(%(length)s)',
3875 'integer': 'INTEGER4',
3876 'bigint': 'BIGINT',
3877 'float': 'FLOAT',
3878 'double': 'FLOAT8',
3879 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3880 'date': 'ANSIDATE',
3881 'time': 'TIME WITHOUT TIME ZONE',
3882 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
3883 'id': 'int not null unique with default next value for %s' % INGRES_SEQNAME,
3884 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3885 'list:integer': 'CLOB',
3886 'list:string': 'CLOB',
3887 'list:reference': 'CLOB',
3888 'big-id': 'bigint not null unique with default next value for %s' % INGRES_SEQNAME,
3889 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3890 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3891 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3892 }
3893
3895 return 'LEFT OUTER JOIN'
3896
3899
3900 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3901 if limitby:
3902 (lmin, lmax) = limitby
3903 fetch_amt = lmax - lmin
3904 if fetch_amt:
3905 sql_s += ' FIRST %d ' % (fetch_amt, )
3906 if lmin:
3907
3908 sql_o += ' OFFSET %d' % (lmin, )
3909 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3910
3911 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3912 credential_decoder=IDENTITY, driver_args={},
3913 adapter_args={}, do_connect=True, after_connection=None):
3914 self.db = db
3915 self.dbengine = "ingres"
3916 self.uri = uri
3917 if do_connect: self.find_driver(adapter_args,uri)
3918 self.pool_size = pool_size
3919 self.folder = folder
3920 self.db_codec = db_codec
3921 self._after_connection = after_connection
3922 self.find_or_make_work_folder()
3923 connstr = self._uri.split(':', 1)[1]
3924
3925 connstr = connstr.lstrip()
3926 while connstr.startswith('/'):
3927 connstr = connstr[1:]
3928 database_name=connstr
3929 vnode = '(local)'
3930 servertype = 'ingres'
3931 trace = (0, None)
3932 driver_args.update(database=database_name,
3933 vnode=vnode,
3934 servertype=servertype,
3935 trace=trace)
3936 def connector(driver_args=driver_args):
3937 return self.driver.connect(**driver_args)
3938 self.connector = connector
3939 if do_connect: self.reconnect()
3940
3942
3943
3944
3945 if hasattr(table,'_primarykey'):
3946 modify_tbl_sql = 'modify %s to btree unique on %s' % \
3947 (table._tablename,
3948 ', '.join(["'%s'" % x for x in table.primarykey]))
3949 self.execute(modify_tbl_sql)
3950 else:
3951 tmp_seqname='%s_iisq' % table._tablename
3952 query=query.replace(INGRES_SEQNAME, tmp_seqname)
3953 self.execute('create sequence %s' % tmp_seqname)
3954 self.execute(query)
3955 self.execute('modify %s to btree unique on %s' % (table._tablename, 'id'))
3956
3957
3959 tmp_seqname='%s_iisq' % table
3960 self.execute('select current value for %s' % tmp_seqname)
3961 return int(self.cursor.fetchone()[0])
3962
3964 return ingresdbi.IntegrityError
3965
3968
3969 drivers = ('ingresdbi',)
3970
3971 types = {
3972 'boolean': 'CHAR(1)',
3973 'string': 'NVARCHAR(%(length)s)',
3974 'text': 'NCLOB',
3975 'json': 'NCLOB',
3976 'password': 'NVARCHAR(%(length)s)',
3977 'blob': 'BLOB',
3978 'upload': 'VARCHAR(%(length)s)',
3979 'integer': 'INTEGER4',
3980 'bigint': 'BIGINT',
3981 'float': 'FLOAT',
3982 'double': 'FLOAT8',
3983 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3984 'date': 'ANSIDATE',
3985 'time': 'TIME WITHOUT TIME ZONE',
3986 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
3987 'id': 'INTEGER4 not null unique with default next value for %s'% INGRES_SEQNAME,
3988 'reference': 'INTEGER4, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3989 'list:integer': 'NCLOB',
3990 'list:string': 'NCLOB',
3991 'list:reference': 'NCLOB',
3992 'big-id': 'BIGINT not null unique with default next value for %s'% INGRES_SEQNAME,
3993 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3994 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3995 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3996 }
3997
3999 drivers = ('sapdb',)
4000
4001 support_distributed_transaction = False
4002 types = {
4003 'boolean': 'CHAR(1)',
4004 'string': 'VARCHAR(%(length)s)',
4005 'text': 'LONG',
4006 'json': 'LONG',
4007 'password': 'VARCHAR(%(length)s)',
4008 'blob': 'LONG',
4009 'upload': 'VARCHAR(%(length)s)',
4010 'integer': 'INT',
4011 'bigint': 'BIGINT',
4012 'float': 'FLOAT',
4013 'double': 'DOUBLE PRECISION',
4014 'decimal': 'FIXED(%(precision)s,%(scale)s)',
4015 'date': 'DATE',
4016 'time': 'TIME',
4017 'datetime': 'TIMESTAMP',
4018 'id': 'INT PRIMARY KEY',
4019 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4020 'list:integer': 'LONG',
4021 'list:string': 'LONG',
4022 'list:reference': 'LONG',
4023 'big-id': 'BIGINT PRIMARY KEY',
4024 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4025 }
4026
4028 return '%s_id_Seq' % table
4029
4030 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
4031 if limitby:
4032 (lmin, lmax) = limitby
4033 if len(sql_w) > 1:
4034 sql_w_row = sql_w + ' AND w_row > %i' % lmin
4035 else:
4036 sql_w_row = 'WHERE w_row > %i' % lmin
4037 return '%s %s FROM (SELECT w_tmp.*, ROWNO w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNO=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
4038 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
4039
4041
4042 self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
4043 self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
4044 % (table._tablename, table._id.name, table._sequence_name))
4045 self.execute(query)
4046
4047 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
4048
4049
4050 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4051 credential_decoder=IDENTITY, driver_args={},
4052 adapter_args={}, do_connect=True, after_connection=None):
4053 self.db = db
4054 self.dbengine = "sapdb"
4055 self.uri = uri
4056 if do_connect: self.find_driver(adapter_args,uri)
4057 self.pool_size = pool_size
4058 self.folder = folder
4059 self.db_codec = db_codec
4060 self._after_connection = after_connection
4061 self.find_or_make_work_folder()
4062 ruri = uri.split('://',1)[1]
4063 m = self.REGEX_URI.match(ruri)
4064 if not m:
4065 raise SyntaxError("Invalid URI string in DAL")
4066 user = credential_decoder(m.group('user'))
4067 if not user:
4068 raise SyntaxError('User required')
4069 password = credential_decoder(m.group('password'))
4070 if not password:
4071 password = ''
4072 host = m.group('host')
4073 if not host:
4074 raise SyntaxError('Host name required')
4075 db = m.group('db')
4076 if not db:
4077 raise SyntaxError('Database name required')
4078 def connector(user=user, password=password, database=db,
4079 host=host, driver_args=driver_args):
4080 return self.driver.Connection(user, password, database,
4081 host, **driver_args)
4082 self.connector = connector
4083 if do_connect: self.reconnect()
4084
4086 self.execute("select %s.NEXTVAL from dual" % table._sequence_name)
4087 return int(self.cursor.fetchone()[0])
4088
4090 drivers = ('cubriddb',)
4091
4092 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
4093
4094 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
4095 credential_decoder=IDENTITY, driver_args={},
4096 adapter_args={}, do_connect=True, after_connection=None):
4097 self.db = db
4098 self.dbengine = "cubrid"
4099 self.uri = uri
4100 if do_connect: self.find_driver(adapter_args,uri)
4101 self.pool_size = pool_size
4102 self.folder = folder
4103 self.db_codec = db_codec
4104 self._after_connection = after_connection
4105 self.find_or_make_work_folder()
4106 ruri = uri.split('://',1)[1]
4107 m = self.REGEX_URI.match(ruri)
4108 if not m:
4109 raise SyntaxError(
4110 "Invalid URI string in DAL: %s" % self.uri)
4111 user = credential_decoder(m.group('user'))
4112 if not user:
4113 raise SyntaxError('User required')
4114 password = credential_decoder(m.group('password'))
4115 if not password:
4116 password = ''
4117 host = m.group('host')
4118 if not host:
4119 raise SyntaxError('Host name required')
4120 db = m.group('db')
4121 if not db:
4122 raise SyntaxError('Database name required')
4123 port = int(m.group('port') or '30000')
4124 charset = m.group('charset') or 'utf8'
4125 user = credential_decoder(user)
4126 passwd = credential_decoder(password)
4127 def connector(host=host,port=port,db=db,
4128 user=user,passwd=password,driver_args=driver_args):
4129 return self.driver.connect(host,port,db,user,passwd,**driver_args)
4130 self.connector = connector
4131 if do_connect: self.reconnect()
4132
4134 self.execute('SET FOREIGN_KEY_CHECKS=1;')
4135 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4136
4141
4142 web2py_filesystem = False
4143
4145 return self.db._adapter.escape(obj)
4146
4148 if not db._adapter.dbengine in ('mysql', 'postgres'):
4149 raise RuntimeError("only MySQL/Postgres can store metadata .table files in database for now")
4150 self.db = db
4151 self.filename = filename
4152 self.mode = mode
4153 if not self.web2py_filesystem:
4154 if db._adapter.dbengine == 'mysql':
4155 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;"
4156 elif db._adapter.dbengine == 'postgres':
4157 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));"
4158 self.db.executesql(sql)
4159 DatabaseStoredFile.web2py_filesystem = True
4160 self.p=0
4161 self.data = ''
4162 if mode in ('r','rw','a'):
4163 query = "SELECT content FROM web2py_filesystem WHERE path='%s'" \
4164 % filename
4165 rows = self.db.executesql(query)
4166 if rows:
4167 self.data = rows[0][0]
4168 elif exists(filename):
4169 datafile = open(filename, 'r')
4170 try:
4171 self.data = datafile.read()
4172 finally:
4173 datafile.close()
4174 elif mode in ('r','rw'):
4175 raise RuntimeError("File %s does not exist" % filename)
4176
4177 - def read(self, bytes):
4178 data = self.data[self.p:self.p+bytes]
4179 self.p += len(data)
4180 return data
4181
4183 i = self.data.find('\n',self.p)+1
4184 if i>0:
4185 data, self.p = self.data[self.p:i], i
4186 else:
4187 data, self.p = self.data[self.p:], len(self.data)
4188 return data
4189
4192
4194 if self.db is not None:
4195 self.db.executesql(
4196 "DELETE FROM web2py_filesystem WHERE path='%s'" % self.filename)
4197 query = "INSERT INTO web2py_filesystem(path,content) VALUES ('%s','%s')"\
4198 % (self.filename, self.data.replace("'","''"))
4199 self.db.executesql(query)
4200 self.db.commit()
4201 self.db = None
4202
4205
4206 @staticmethod
4208 if exists(filename):
4209 return True
4210 query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename
4211 if db.executesql(query):
4212 return True
4213 return False
4214
4217
4220
4221 - def file_open(self, filename, mode='rb', lock=True):
4223
4226
4228 query = "DELETE FROM web2py_filesystem WHERE path='%s'" % filename
4229 self.db.executesql(query)
4230 self.db.commit()
4231
4233 uploads_in_blob = True
4234
4235 REGEX_URI = re.compile('^(?P<instance>.*)/(?P<db>.*)$')
4236
4237 - def __init__(self, db, uri='google:sql://realm:domain/database',
4238 pool_size=0, folder=None, db_codec='UTF-8',
4239 credential_decoder=IDENTITY, driver_args={},
4240 adapter_args={}, do_connect=True, after_connection=None):
4241
4242 self.db = db
4243 self.dbengine = "mysql"
4244 self.uri = uri
4245 self.pool_size = pool_size
4246 self.db_codec = db_codec
4247 self._after_connection = after_connection
4248 self.folder = folder or pjoin('$HOME',THREAD_LOCAL.folder.split(
4249 os.sep+'applications'+os.sep,1)[1])
4250 ruri = uri.split("://")[1]
4251 m = self.REGEX_URI.match(ruri)
4252 if not m:
4253 raise SyntaxError("Invalid URI string in SQLDB: %s" % self.uri)
4254 instance = credential_decoder(m.group('instance'))
4255 self.dbstring = db = credential_decoder(m.group('db'))
4256 driver_args['instance'] = instance
4257 if not 'charset' in driver_args:
4258 driver_args['charset'] = 'utf8'
4259 self.createdb = createdb = adapter_args.get('createdb',True)
4260 if not createdb:
4261 driver_args['database'] = db
4262 def connector(driver_args=driver_args):
4263 return rdbms.connect(**driver_args)
4264 self.connector = connector
4265 if do_connect: self.reconnect()
4266
4268 if self.createdb:
4269
4270 self.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbstring)
4271 self.execute('USE %s' % self.dbstring)
4272 self.execute("SET FOREIGN_KEY_CHECKS=1;")
4273 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4274
4275 - def execute(self, command, *a, **b):
4277
4279 can_select_for_update = False
4280
4281 @staticmethod
4283 if isinstance(obj, str):
4284 return obj.decode('utf8')
4285 elif not isinstance(obj, unicode):
4286 return unicode(obj)
4287 return obj
4288
4290 return table._id > 0
4291
4293 field_is_type = fieldtype.startswith
4294 if isinstance(obj, CALLABLETYPES):
4295 obj = obj()
4296 if isinstance(fieldtype, SQLCustomType):
4297 return fieldtype.encoder(obj)
4298 if isinstance(obj, (Expression, Field)):
4299 raise SyntaxError("non supported on GAE")
4300 if self.dbengine == 'google:datastore':
4301 if isinstance(fieldtype, gae.Property):
4302 return obj
4303 is_string = isinstance(fieldtype,str)
4304 is_list = is_string and field_is_type('list:')
4305 if is_list:
4306 if not obj:
4307 obj = []
4308 if not isinstance(obj, (list, tuple)):
4309 obj = [obj]
4310 if obj == '' and not \
4311 (is_string and fieldtype[:2] in ['st','te', 'pa','up']):
4312 return None
4313 if not obj is None:
4314 if isinstance(obj, list) and not is_list:
4315 obj = [self.represent(o, fieldtype) for o in obj]
4316 elif fieldtype in ('integer','bigint','id'):
4317 obj = long(obj)
4318 elif fieldtype == 'double':
4319 obj = float(obj)
4320 elif is_string and field_is_type('reference'):
4321 if isinstance(obj, (Row, Reference)):
4322 obj = obj['id']
4323 obj = long(obj)
4324 elif fieldtype == 'boolean':
4325 if obj and not str(obj)[0].upper() in '0F':
4326 obj = True
4327 else:
4328 obj = False
4329 elif fieldtype == 'date':
4330 if not isinstance(obj, datetime.date):
4331 (y, m, d) = map(int,str(obj).strip().split('-'))
4332 obj = datetime.date(y, m, d)
4333 elif isinstance(obj,datetime.datetime):
4334 (y, m, d) = (obj.year, obj.month, obj.day)
4335 obj = datetime.date(y, m, d)
4336 elif fieldtype == 'time':
4337 if not isinstance(obj, datetime.time):
4338 time_items = map(int,str(obj).strip().split(':')[:3])
4339 if len(time_items) == 3:
4340 (h, mi, s) = time_items
4341 else:
4342 (h, mi, s) = time_items + [0]
4343 obj = datetime.time(h, mi, s)
4344 elif fieldtype == 'datetime':
4345 if not isinstance(obj, datetime.datetime):
4346 (y, m, d) = map(int,str(obj)[:10].strip().split('-'))
4347 time_items = map(int,str(obj)[11:].strip().split(':')[:3])
4348 while len(time_items)<3:
4349 time_items.append(0)
4350 (h, mi, s) = time_items
4351 obj = datetime.datetime(y, m, d, h, mi, s)
4352 elif fieldtype == 'blob':
4353 pass
4354 elif fieldtype == 'json':
4355 obj = self.to_unicode(obj)
4356 if have_serializers:
4357 obj = serializers.loads_json(obj)
4358 elif simplejson:
4359 obj = simplejson.loads(obj)
4360 else:
4361 raise RuntimeError("missing simplejson")
4362 elif is_string and field_is_type('list:string'):
4363 return map(self.to_unicode,obj)
4364 elif is_list:
4365 return map(int,obj)
4366 else:
4367 obj = self.to_unicode(obj)
4368 return obj
4369
4371 return 'insert %s in %s' % (fields, table)
4372
4373 - def _count(self,query,distinct=None):
4374 return 'count %s' % repr(query)
4375
4376 - def _select(self,query,fields,attributes):
4377 return 'select %s where %s' % (repr(fields), repr(query))
4378
4379 - def _delete(self,tablename, query):
4380 return 'delete %s where %s' % (repr(tablename),repr(query))
4381
4382 - def _update(self,tablename,query,fields):
4383 return 'update %s (%s) where %s' % (repr(tablename),
4384 repr(fields),repr(query))
4385
4387 """
4388 remember: no transactions on many NoSQL
4389 """
4390 pass
4391
4393 """
4394 remember: no transactions on many NoSQL
4395 """
4396 pass
4397
4399 """
4400 remember: no transactions on many NoSQL
4401 """
4402 pass
4403
4404
4405
4406 - def OR(self,first,second): raise SyntaxError("Not supported")
4407 - def AND(self,first,second): raise SyntaxError("Not supported")
4408 - def AS(self,first,second): raise SyntaxError("Not supported")
4409 - def ON(self,first,second): raise SyntaxError("Not supported")
4410 - def STARTSWITH(self,first,second=None): raise SyntaxError("Not supported")
4411 - def ENDSWITH(self,first,second=None): raise SyntaxError("Not supported")
4412 - def ADD(self,first,second): raise SyntaxError("Not supported")
4413 - def SUB(self,first,second): raise SyntaxError("Not supported")
4414 - def MUL(self,first,second): raise SyntaxError("Not supported")
4415 - def DIV(self,first,second): raise SyntaxError("Not supported")
4416 - def LOWER(self,first): raise SyntaxError("Not supported")
4417 - def UPPER(self,first): raise SyntaxError("Not supported")
4419 - def AGGREGATE(self,first,what): raise SyntaxError("Not supported")
4420 - def LEFT_JOIN(self): raise SyntaxError("Not supported")
4421 - def RANDOM(self): raise SyntaxError("Not supported")
4422 - def SUBSTRING(self,field,parameters): raise SyntaxError("Not supported")
4423 - def PRIMARY_KEY(self,key): raise SyntaxError("Not supported")
4424 - def ILIKE(self,first,second): raise SyntaxError("Not supported")
4425 - def drop(self,table,mode): raise SyntaxError("Not supported")
4426 - def alias(self,table,alias): raise SyntaxError("Not supported")
4427 - def migrate_table(self,*a,**b): raise SyntaxError("Not supported")
4429 - def prepare(self,key): raise SyntaxError("Not supported")
4432 - def concat_add(self,table): raise SyntaxError("Not supported")
4433 - def constraint_name(self, table, fieldname): raise SyntaxError("Not supported")
4435 - def log_execute(self,*a,**b): raise SyntaxError("Not supported")
4436 - def execute(self,*a,**b): raise SyntaxError("Not supported")
4438 - def lastrowid(self,table): raise SyntaxError("Not supported")
4440 - def rowslice(self,rows,minimum=0,maximum=None): raise SyntaxError("Not supported")
4441
4442
4443 -class GAEF(object):
4444 - def __init__(self,name,op,value,apply):
4445 self.name=name=='id' and '__key__' or name
4446 self.op=op
4447 self.value=value
4448 self.apply=apply
4450 return '(%s %s %s:%s)' % (self.name, self.op, repr(self.value), type(self.value))
4451
4453 uploads_in_blob = True
4454 types = {}
4455
4457 - def file_open(self, filename, mode='rb', lock=True): pass
4459
4460 REGEX_NAMESPACE = re.compile('.*://(?P<namespace>.+)')
4461
4462 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4463 credential_decoder=IDENTITY, driver_args={},
4464 adapter_args={}, do_connect=True, after_connection=None):
4465 self.types.update({
4466 'boolean': gae.BooleanProperty,
4467 'string': (lambda: gae.StringProperty(multiline=True)),
4468 'text': gae.TextProperty,
4469 'json': gae.TextProperty,
4470 'password': gae.StringProperty,
4471 'blob': gae.BlobProperty,
4472 'upload': gae.StringProperty,
4473 'integer': gae.IntegerProperty,
4474 'bigint': gae.IntegerProperty,
4475 'float': gae.FloatProperty,
4476 'double': gae.FloatProperty,
4477 'decimal': GAEDecimalProperty,
4478 'date': gae.DateProperty,
4479 'time': gae.TimeProperty,
4480 'datetime': gae.DateTimeProperty,
4481 'id': None,
4482 'reference': gae.IntegerProperty,
4483 'list:string': (lambda: gae.StringListProperty(default=None)),
4484 'list:integer': (lambda: gae.ListProperty(int,default=None)),
4485 'list:reference': (lambda: gae.ListProperty(int,default=None)),
4486 })
4487 self.db = db
4488 self.uri = uri
4489 self.dbengine = 'google:datastore'
4490 self.folder = folder
4491 db['_lastsql'] = ''
4492 self.db_codec = 'UTF-8'
4493 self._after_connection = after_connection
4494 self.pool_size = 0
4495 match = self.REGEX_NAMESPACE.match(uri)
4496 if match:
4497 namespace_manager.set_namespace(match.group('namespace'))
4498
4499 - def parse_id(self, value, field_type):
4501
4502 - def create_table(self,table,migrate=True,fake_migrate=False, polymodel=None):
4503 myfields = {}
4504 for field in table:
4505 if isinstance(polymodel,Table) and field.name in polymodel.fields():
4506 continue
4507 attr = {}
4508 if isinstance(field.custom_qaulifier, dict):
4509
4510 attr = field.custom_qualifier
4511 field_type = field.type
4512 if isinstance(field_type, SQLCustomType):
4513 ftype = self.types[field_type.native or field_type.type](**attr)
4514 elif isinstance(field_type, gae.Property):
4515 ftype = field_type
4516 elif field_type.startswith('id'):
4517 continue
4518 elif field_type.startswith('decimal'):
4519 precision, scale = field_type[7:].strip('()').split(',')
4520 precision = int(precision)
4521 scale = int(scale)
4522 ftype = GAEDecimalProperty(precision, scale, **attr)
4523 elif field_type.startswith('reference'):
4524 if field.notnull:
4525 attr = dict(required=True)
4526 referenced = field_type[10:].strip()
4527 ftype = self.types[field_type[:9]](referenced, **attr)
4528 elif field_type.startswith('list:reference'):
4529 if field.notnull:
4530 attr['required'] = True
4531 referenced = field_type[15:].strip()
4532 ftype = self.types[field_type[:14]](**attr)
4533 elif field_type.startswith('list:'):
4534 ftype = self.types[field_type](**attr)
4535 elif not field_type in self.types\
4536 or not self.types[field_type]:
4537 raise SyntaxError('Field: unknown field type: %s' % field_type)
4538 else:
4539 ftype = self.types[field_type](**attr)
4540 myfields[field.name] = ftype
4541 if not polymodel:
4542 table._tableobj = classobj(table._tablename, (gae.Model, ), myfields)
4543 elif polymodel==True:
4544 table._tableobj = classobj(table._tablename, (PolyModel, ), myfields)
4545 elif isinstance(polymodel,Table):
4546 table._tableobj = classobj(table._tablename, (polymodel._tableobj, ), myfields)
4547 else:
4548 raise SyntaxError("polymodel must be None, True, a table or a tablename")
4549 return None
4550
4551 - def expand(self,expression,field_type=None):
4552 if isinstance(expression,Field):
4553 if expression.type in ('text', 'blob', 'json'):
4554 raise SyntaxError('AppEngine does not index by: %s' % expression.type)
4555 return expression.name
4556 elif isinstance(expression, (Expression, Query)):
4557 if not expression.second is None:
4558 return expression.op(expression.first, expression.second)
4559 elif not expression.first is None:
4560 return expression.op(expression.first)
4561 else:
4562 return expression.op()
4563 elif field_type:
4564 return self.represent(expression,field_type)
4565 elif isinstance(expression,(list,tuple)):
4566 return ','.join([self.represent(item,field_type) for item in expression])
4567 else:
4568 return str(expression)
4569
4570
4571 - def AND(self,first,second):
4577
4578 - def EQ(self,first,second=None):
4579 if isinstance(second, Key):
4580 return [GAEF(first.name,'=',second,lambda a,b:a==b)]
4581 return [GAEF(first.name,'=',self.represent(second,first.type),lambda a,b:a==b)]
4582
4583 - def NE(self,first,second=None):
4584 if first.type != 'id':
4585 return [GAEF(first.name,'!=',self.represent(second,first.type),lambda a,b:a!=b)]
4586 else:
4587 if not second is None:
4588 second = Key.from_path(first._tablename, long(second))
4589 return [GAEF(first.name,'!=',second,lambda a,b:a!=b)]
4590
4591 - def LT(self,first,second=None):
4592 if first.type != 'id':
4593 return [GAEF(first.name,'<',self.represent(second,first.type),lambda a,b:a<b)]
4594 else:
4595 second = Key.from_path(first._tablename, long(second))
4596 return [GAEF(first.name,'<',second,lambda a,b:a<b)]
4597
4598 - def LE(self,first,second=None):
4599 if first.type != 'id':
4600 return [GAEF(first.name,'<=',self.represent(second,first.type),lambda a,b:a<=b)]
4601 else:
4602 second = Key.from_path(first._tablename, long(second))
4603 return [GAEF(first.name,'<=',second,lambda a,b:a<=b)]
4604
4605 - def GT(self,first,second=None):
4606 if first.type != 'id' or second==0 or second == '0':
4607 return [GAEF(first.name,'>',self.represent(second,first.type),lambda a,b:a>b)]
4608 else:
4609 second = Key.from_path(first._tablename, long(second))
4610 return [GAEF(first.name,'>',second,lambda a,b:a>b)]
4611
4612 - def GE(self,first,second=None):
4613 if first.type != 'id':
4614 return [GAEF(first.name,'>=',self.represent(second,first.type),lambda a,b:a>=b)]
4615 else:
4616 second = Key.from_path(first._tablename, long(second))
4617 return [GAEF(first.name,'>=',second,lambda a,b:a>=b)]
4618
4621
4622 - def COMMA(self,first,second):
4624
4625 - def BELONGS(self,first,second=None):
4626 if not isinstance(second,(list, tuple)):
4627 raise SyntaxError("Not supported")
4628 if first.type != 'id':
4629 return [GAEF(first.name,'in',self.represent(second,first.type),lambda a,b:a in b)]
4630 else:
4631 second = [Key.from_path(first._tablename, int(i)) for i in second]
4632 return [GAEF(first.name,'in',second,lambda a,b:a in b)]
4633
4634 - def CONTAINS(self,first,second,case_sensitive=False):
4639
4640 - def NOT(self,first):
4641 nops = { self.EQ: self.NE,
4642 self.NE: self.EQ,
4643 self.LT: self.GE,
4644 self.GT: self.LE,
4645 self.LE: self.GT,
4646 self.GE: self.LT}
4647 if not isinstance(first,Query):
4648 raise SyntaxError("Not suported")
4649 nop = nops.get(first.op,None)
4650 if not nop:
4651 raise SyntaxError("Not suported %s" % first.op.__name__)
4652 first.op = nop
4653 return self.expand(first)
4654
4657
4658 - def select_raw(self,query,fields=None,attributes=None):
4659 db = self.db
4660 fields = fields or []
4661 attributes = attributes or {}
4662 args_get = attributes.get
4663 new_fields = []
4664 for item in fields:
4665 if isinstance(item,SQLALL):
4666 new_fields += item._table
4667 else:
4668 new_fields.append(item)
4669 fields = new_fields
4670 if query:
4671 tablename = self.get_table(query)
4672 elif fields:
4673 tablename = fields[0].tablename
4674 query = db._adapter.id_query(fields[0].table)
4675 else:
4676 raise SyntaxError("Unable to determine a tablename")
4677
4678 if query:
4679 if use_common_filters(query):
4680 query = self.common_filter(query,[tablename])
4681
4682
4683 tableobj = db[tablename]._tableobj
4684 filters = self.expand(query)
4685
4686 projection = None
4687 if len(db[tablename].fields) == len(fields):
4688
4689 projection = None
4690 elif args_get('projection') == True:
4691 projection = []
4692 for f in fields:
4693 if f.type in ['text', 'blob', 'json']:
4694 raise SyntaxError(
4695 "text and blob field types not allowed in projection queries")
4696 else:
4697 projection.append(f.name)
4698
4699
4700
4701 query_projection = [
4702 p for p in projection if \
4703 p != db[tablename]._id.name] if projection \
4704 else None
4705
4706 cursor = None
4707 if isinstance(args_get('reusecursor'), str):
4708 cursor = args_get('reusecursor')
4709 items = gae.Query(tableobj, projection=query_projection,
4710 cursor=cursor)
4711
4712 for filter in filters:
4713 if args_get('projection') == True and \
4714 filter.name in query_projection and \
4715 filter.op in ['=', '<=', '>=']:
4716 raise SyntaxError(
4717 "projection fields cannot have equality filters")
4718 if filter.name=='__key__' and filter.op=='>' and filter.value==0:
4719 continue
4720 elif filter.name=='__key__' and filter.op=='=':
4721 if filter.value==0:
4722 items = []
4723 elif isinstance(filter.value, Key):
4724
4725
4726
4727 item = tableobj.get(filter.value)
4728 items = (item and [item]) or []
4729 else:
4730
4731
4732
4733 item = tableobj.get_by_id(filter.value)
4734 items = (item and [item]) or []
4735 elif isinstance(items,list):
4736 items = [i for i in items if filter.apply(
4737 getattr(item,filter.name),filter.value)]
4738 else:
4739 if filter.name=='__key__' and filter.op != 'in':
4740 items.order('__key__')
4741 items = items.filter('%s %s' % (filter.name,filter.op),
4742 filter.value)
4743 if not isinstance(items,list):
4744 if args_get('left', None):
4745 raise SyntaxError('Set: no left join in appengine')
4746 if args_get('groupby', None):
4747 raise SyntaxError('Set: no groupby in appengine')
4748 orderby = args_get('orderby', False)
4749 if orderby:
4750
4751 if isinstance(orderby, (list, tuple)):
4752 orderby = xorify(orderby)
4753 if isinstance(orderby,Expression):
4754 orderby = self.expand(orderby)
4755 orders = orderby.split(', ')
4756 for order in orders:
4757 order={'-id':'-__key__','id':'__key__'}.get(order,order)
4758 items = items.order(order)
4759 if args_get('limitby', None):
4760 (lmin, lmax) = attributes['limitby']
4761 (limit, offset) = (lmax - lmin, lmin)
4762 rows = items.fetch(limit,offset=offset)
4763
4764
4765 if args_get('reusecursor'):
4766 db['_lastcursor'] = items.cursor()
4767 items = rows
4768 return (items, tablename, projection or db[tablename].fields)
4769
4770 - def select(self,query,fields,attributes):
4771 """
4772 This is the GAE version of select. some notes to consider:
4773 - db['_lastsql'] is not set because there is not SQL statement string
4774 for a GAE query
4775 - 'nativeRef' is a magical fieldname used for self references on GAE
4776 - optional attribute 'projection' when set to True will trigger
4777 use of the GAE projection queries. note that there are rules for
4778 what is accepted imposed by GAE: each field must be indexed,
4779 projection queries cannot contain blob or text fields, and you
4780 cannot use == and also select that same field. see https://developers.google.com/appengine/docs/python/datastore/queries#Query_Projection
4781 - optional attribute 'reusecursor' allows use of cursor with queries
4782 that have the limitby attribute. Set the attribute to True for the
4783 first query, set it to the value of db['_lastcursor'] to continue
4784 a previous query. The user must save the cursor value between
4785 requests, and the filters must be identical. It is up to the user
4786 to follow google's limitations: https://developers.google.com/appengine/docs/python/datastore/queries#Query_Cursors
4787 """
4788
4789 (items, tablename, fields) = self.select_raw(query,fields,attributes)
4790
4791 rows = [[(t==self.db[tablename]._id.name and item) or \
4792 (t=='nativeRef' and item) or getattr(item, t) \
4793 for t in fields] for item in items]
4794 colnames = ['%s.%s' % (tablename, t) for t in fields]
4795 processor = attributes.get('processor',self.parse)
4796 return processor(rows,fields,colnames,False)
4797
4798 - def count(self,query,distinct=None,limit=None):
4799 if distinct:
4800 raise RuntimeError("COUNT DISTINCT not supported")
4801 (items, tablename, fields) = self.select_raw(query)
4802
4803 try:
4804 return len(items)
4805 except TypeError:
4806 return items.count(limit=limit)
4807
4808 - def delete(self,tablename, query):
4809 """
4810 This function was changed on 2010-05-04 because according to
4811 http://code.google.com/p/googleappengine/issues/detail?id=3119
4812 GAE no longer supports deleting more than 1000 records.
4813 """
4814
4815 (items, tablename, fields) = self.select_raw(query)
4816
4817 if not isinstance(items,list):
4818
4819
4820 leftitems = items.fetch(1000, keys_only=True)
4821 counter = 0
4822 while len(leftitems):
4823 counter += len(leftitems)
4824 gae.delete(leftitems)
4825 leftitems = items.fetch(1000, keys_only=True)
4826 else:
4827 counter = len(items)
4828 gae.delete(items)
4829 return counter
4830
4831 - def update(self,tablename,query,update_fields):
4832
4833 (items, tablename, fields) = self.select_raw(query)
4834 counter = 0
4835 for item in items:
4836 for field, value in update_fields:
4837 setattr(item, field.name, self.represent(value,field.type))
4838 item.put()
4839 counter += 1
4840 LOGGER.info(str(counter))
4841 return counter
4842
4843 - def insert(self,table,fields):
4844 dfields=dict((f.name,self.represent(v,f.type)) for f,v in fields)
4845
4846 tmp = table._tableobj(**dfields)
4847 tmp.put()
4848 rid = Reference(tmp.key().id())
4849 (rid._table, rid._record, rid._gaekey) = (table, None, tmp.key())
4850 return rid
4851
4853 parsed_items = []
4854 for item in items:
4855 dfields=dict((f.name,self.represent(v,f.type)) for f,v in item)
4856 parsed_items.append(table._tableobj(**dfields))
4857 gae.put(parsed_items)
4858 return True
4859
4861 return uuid.UUID(uuidv).int
4862
4864 return str(uuid.UUID(int=n))
4865
4867 drivers = ('couchdb',)
4868
4869 uploads_in_blob = True
4870 types = {
4871 'boolean': bool,
4872 'string': str,
4873 'text': str,
4874 'json': str,
4875 'password': str,
4876 'blob': str,
4877 'upload': str,
4878 'integer': long,
4879 'bigint': long,
4880 'float': float,
4881 'double': float,
4882 'date': datetime.date,
4883 'time': datetime.time,
4884 'datetime': datetime.datetime,
4885 'id': long,
4886 'reference': long,
4887 'list:string': list,
4888 'list:integer': list,
4889 'list:reference': list,
4890 }
4891
4893 - def file_open(self, filename, mode='rb', lock=True): pass
4895
4896 - def expand(self,expression,field_type=None):
4897 if isinstance(expression,Field):
4898 if expression.type=='id':
4899 return "%s._id" % expression.tablename
4900 return BaseAdapter.expand(self,expression,field_type)
4901
4902 - def AND(self,first,second):
4904
4905 - def OR(self,first,second):
4907
4908 - def EQ(self,first,second):
4912
4913 - def NE(self,first,second):
4917
4918 - def COMMA(self,first,second):
4920
4922 value = NoSQLAdapter.represent(self, obj, fieldtype)
4923 if fieldtype=='id':
4924 return repr(str(int(value)))
4925 elif fieldtype in ('date','time','datetime','boolean'):
4926 return serializers.json(value)
4927 return repr(not isinstance(value,unicode) and value \
4928 or value and value.encode('utf8'))
4929
4930 - def __init__(self,db,uri='couchdb://127.0.0.1:5984',
4931 pool_size=0,folder=None,db_codec ='UTF-8',
4932 credential_decoder=IDENTITY, driver_args={},
4933 adapter_args={}, do_connect=True, after_connection=None):
4934 self.db = db
4935 self.uri = uri
4936 if do_connect: self.find_driver(adapter_args)
4937 self.dbengine = 'couchdb'
4938 self.folder = folder
4939 db['_lastsql'] = ''
4940 self.db_codec = 'UTF-8'
4941 self._after_connection = after_connection
4942 self.pool_size = pool_size
4943
4944 url='http://'+uri[10:]
4945 def connector(url=url,driver_args=driver_args):
4946 return self.driver.Server(url,**driver_args)
4947 self.reconnect(connector,cursor=False)
4948
4949 - def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None):
4950 if migrate:
4951 try:
4952 self.connection.create(table._tablename)
4953 except:
4954 pass
4955
4956 - def insert(self,table,fields):
4963
4964 - def _select(self,query,fields,attributes):
4965 if not isinstance(query,Query):
4966 raise SyntaxError("Not Supported")
4967 for key in set(attributes.keys())-SELECT_ARGS:
4968 raise SyntaxError('invalid select attribute: %s' % key)
4969 new_fields=[]
4970 for item in fields:
4971 if isinstance(item,SQLALL):
4972 new_fields += item._table
4973 else:
4974 new_fields.append(item)
4975 def uid(fd):
4976 return fd=='id' and '_id' or fd
4977 def get(row,fd):
4978 return fd=='id' and int(row['_id']) or row.get(fd,None)
4979 fields = new_fields
4980 tablename = self.get_table(query)
4981 fieldnames = [f.name for f in (fields or self.db[tablename])]
4982 colnames = ['%s.%s' % (tablename,k) for k in fieldnames]
4983 fields = ','.join(['%s.%s' % (tablename,uid(f)) for f in fieldnames])
4984 fn="(function(%(t)s){if(%(query)s)emit(%(order)s,[%(fields)s]);})" %\
4985 dict(t=tablename,
4986 query=self.expand(query),
4987 order='%s._id' % tablename,
4988 fields=fields)
4989 return fn, colnames
4990
4991 - def select(self,query,fields,attributes):
4992 if not isinstance(query,Query):
4993 raise SyntaxError("Not Supported")
4994 fn, colnames = self._select(query,fields,attributes)
4995 tablename = colnames[0].split('.')[0]
4996 ctable = self.connection[tablename]
4997 rows = [cols['value'] for cols in ctable.query(fn)]
4998 processor = attributes.get('processor',self.parse)
4999 return processor(rows,fields,colnames,False)
5000
5001 - def delete(self,tablename,query):
5002 if not isinstance(query,Query):
5003 raise SyntaxError("Not Supported")
5004 if query.first.type=='id' and query.op==self.EQ:
5005 id = query.second
5006 tablename = query.first.tablename
5007 assert(tablename == query.first.tablename)
5008 ctable = self.connection[tablename]
5009 try:
5010 del ctable[str(id)]
5011 return 1
5012 except couchdb.http.ResourceNotFound:
5013 return 0
5014 else:
5015 tablename = self.get_table(query)
5016 rows = self.select(query,[self.db[tablename]._id],{})
5017 ctable = self.connection[tablename]
5018 for row in rows:
5019 del ctable[str(row.id)]
5020 return len(rows)
5021
5022 - def update(self,tablename,query,fields):
5023 if not isinstance(query,Query):
5024 raise SyntaxError("Not Supported")
5025 if query.first.type=='id' and query.op==self.EQ:
5026 id = query.second
5027 tablename = query.first.tablename
5028 ctable = self.connection[tablename]
5029 try:
5030 doc = ctable[str(id)]
5031 for key,value in fields:
5032 doc[key.name] = self.represent(value,self.db[tablename][key.name].type)
5033 ctable.save(doc)
5034 return 1
5035 except couchdb.http.ResourceNotFound:
5036 return 0
5037 else:
5038 tablename = self.get_table(query)
5039 rows = self.select(query,[self.db[tablename]._id],{})
5040 ctable = self.connection[tablename]
5041 table = self.db[tablename]
5042 for row in rows:
5043 doc = ctable[str(row.id)]
5044 for key,value in fields:
5045 doc[key.name] = self.represent(value,table[key.name].type)
5046 ctable.save(doc)
5047 return len(rows)
5048
5049 - def count(self,query,distinct=None):
5050 if distinct:
5051 raise RuntimeError("COUNT DISTINCT not supported")
5052 if not isinstance(query,Query):
5053 raise SyntaxError("Not Supported")
5054 tablename = self.get_table(query)
5055 rows = self.select(query,[self.db[tablename]._id],{})
5056 return len(rows)
5057
5059 """
5060 validates that the given text is clean: only contains [0-9a-zA-Z_]
5061 """
5062 if not REGEX_ALPHANUMERIC.match(text):
5063 raise SyntaxError('invalid table or field name: %s' % text)
5064 return text
5065
5067 native_json = True
5068 drivers = ('pymongo',)
5069
5070 uploads_in_blob = True
5071
5072 types = {
5073 'boolean': bool,
5074 'string': str,
5075 'text': str,
5076 'json': str,
5077 'password': str,
5078 'blob': str,
5079 'upload': str,
5080 'integer': long,
5081 'bigint': long,
5082 'float': float,
5083 'double': float,
5084 'date': datetime.date,
5085 'time': datetime.time,
5086 'datetime': datetime.datetime,
5087 'id': long,
5088 'reference': long,
5089 'list:string': list,
5090 'list:integer': list,
5091 'list:reference': list,
5092 }
5093
5094 error_messages = {"javascript_needed": "This must yet be replaced" +
5095 " with javascript in order to work."}
5096
5097 - def __init__(self,db,uri='mongodb://127.0.0.1:5984/db',
5098 pool_size=0, folder=None, db_codec ='UTF-8',
5099 credential_decoder=IDENTITY, driver_args={},
5100 adapter_args={}, do_connect=True, after_connection=None):
5101
5102 self.db = db
5103 self.uri = uri
5104 if do_connect: self.find_driver(adapter_args)
5105 import random
5106 from bson.objectid import ObjectId
5107 from bson.son import SON
5108 import pymongo.uri_parser
5109
5110 m = pymongo.uri_parser.parse_uri(uri)
5111
5112 self.SON = SON
5113 self.ObjectId = ObjectId
5114 self.random = random
5115
5116 self.dbengine = 'mongodb'
5117 self.folder = folder
5118 db['_lastsql'] = ''
5119 self.db_codec = 'UTF-8'
5120 self._after_connection = after_connection
5121 self.pool_size = pool_size
5122
5123
5124 self.minimumreplication = adapter_args.get('minimumreplication',0)
5125
5126
5127
5128
5129 self.safe = adapter_args.get('safe',True)
5130
5131 if isinstance(m,tuple):
5132 m = {"database" : m[1]}
5133 if m.get('database')==None:
5134 raise SyntaxError("Database is required!")
5135 def connector(uri=self.uri,m=m):
5136 try:
5137
5138 if hasattr(self.driver, "MongoClient"):
5139 Connection = self.driver.MongoClient
5140 else:
5141 Connection = self.driver.Connection
5142 return Connection(uri)[m.get('database')]
5143 except self.driver.errors.ConnectionFailure:
5144 inst = sys.exc_info()[1]
5145 raise SyntaxError("The connection to " +
5146 uri + " could not be made")
5147
5148 self.reconnect(connector,cursor=False)
5149
5151 """ Convert input to a valid Mongodb ObjectId instance
5152
5153 self.object_id("<random>") -> ObjectId (not unique) instance """
5154 if not arg:
5155 arg = 0
5156 if isinstance(arg, basestring):
5157
5158 rawhex = len(arg.replace("0x", "").replace("L", "")) == 24
5159 if arg.isdigit() and (not rawhex):
5160 arg = int(arg)
5161 elif arg == "<random>":
5162 arg = int("0x%sL" % \
5163 "".join([self.random.choice("0123456789abcdef") \
5164 for x in range(24)]), 0)
5165 elif arg.isalnum():
5166 if not arg.startswith("0x"):
5167 arg = "0x%s" % arg
5168 try:
5169 arg = int(arg, 0)
5170 except ValueError, e:
5171 raise ValueError(
5172 "invalid objectid argument string: %s" % e)
5173 else:
5174 raise ValueError("Invalid objectid argument string. " +
5175 "Requires an integer or base 16 value")
5176 elif isinstance(arg, self.ObjectId):
5177 return arg
5178 if not isinstance(arg, (int, long)):
5179 raise TypeError("object_id argument must be of type " +
5180 "ObjectId or an objectid representable integer")
5181 if arg == 0:
5182 hexvalue = "".zfill(24)
5183 else:
5184 hexvalue = hex(arg)[2:].replace("L", "")
5185 return self.ObjectId(hexvalue)
5186
5188 value = NoSQLAdapter.represent(self, obj, fieldtype)
5189 if fieldtype =='date':
5190 if value == None:
5191 return value
5192
5193 t = datetime.time(0, 0, 0)
5194
5195
5196 return datetime.datetime.combine(value, t)
5197 elif fieldtype == 'time':
5198 if value == None:
5199 return value
5200
5201 d = datetime.date(2000, 1, 1)
5202
5203
5204 return datetime.datetime.combine(d, value)
5205 elif fieldtype == 'list:string' or \
5206 fieldtype == 'list:integer' or \
5207 fieldtype == 'list:reference':
5208 return value
5209 return value
5210
5211
5212
5213
5214 - def insert(self, table, fields, safe=None):
5215 if safe==None:
5216 safe = self.safe
5217 ctable = self.connection[table._tablename]
5218 values = dict()
5219 for k, v in fields:
5220 if not k.name in ["id", "safe"]:
5221 fieldname = k.name
5222 fieldtype = table[k.name].type
5223 if ("reference" in fieldtype) or (fieldtype=="id"):
5224 values[fieldname] = self.object_id(v)
5225 else:
5226 values[fieldname] = self.represent(v, fieldtype)
5227 ctable.insert(values, safe=safe)
5228 return int(str(values['_id']), 16)
5229
5230 - def create_table(self, table, migrate=True, fake_migrate=False,
5231 polymodel=None, isCapped=False):
5232 if isCapped:
5233 raise RuntimeError("Not implemented")
5234
5235 - def count(self, query, distinct=None, snapshot=True):
5236 if distinct:
5237 raise RuntimeError("COUNT DISTINCT not supported")
5238 if not isinstance(query,Query):
5239 raise SyntaxError("Not Supported")
5240 tablename = self.get_table(query)
5241 return int(self.select(query,[self.db[tablename]._id], {},
5242 count=True,snapshot=snapshot)['count'])
5243
5244
5245
5246
5247
5248 - def expand(self, expression, field_type=None):
5249 if isinstance(expression, Query):
5250
5251
5252
5253
5254
5255 if isinstance(expression.first,Field) and \
5256 ((expression.first.type == 'id') or \
5257 ("reference" in expression.first.type)):
5258 if expression.first.type == 'id':
5259 expression.first.name = '_id'
5260
5261 expression.second = self.object_id(expression.second)
5262 result = expression.op(expression.first, expression.second)
5263 if isinstance(expression, Field):
5264 if expression.type=='id':
5265 result = "_id"
5266 else:
5267 result = expression.name
5268
5269 elif isinstance(expression, (Expression, Query)):
5270 if not expression.second is None:
5271 result = expression.op(expression.first, expression.second)
5272 elif not expression.first is None:
5273 result = expression.op(expression.first)
5274 elif not isinstance(expression.op, str):
5275 result = expression.op()
5276 else:
5277 result = expression.op
5278 elif field_type:
5279 result = str(self.represent(expression,field_type))
5280 elif isinstance(expression,(list,tuple)):
5281 result = ','.join(self.represent(item,field_type) for
5282 item in expression)
5283 else:
5284 result = expression
5285 return result
5286
5287 - def _select(self, query, fields, attributes):
5288 if 'for_update' in attributes:
5289 logging.warn('mongodb does not support for_update')
5290 for key in set(attributes.keys())-set(('limitby',
5291 'orderby','for_update')):
5292 if attributes[key]!=None:
5293 logging.warn('select attribute not implemented: %s' % key)
5294
5295 new_fields=[]
5296 mongosort_list = []
5297
5298
5299 orderby = attributes.get('orderby', False)
5300 limitby = attributes.get('limitby', False)
5301
5302 if orderby:
5303 if isinstance(orderby, (list, tuple)):
5304 orderby = xorify(orderby)
5305
5306
5307 for f in self.expand(orderby).split(','):
5308 if f.startswith('-'):
5309 mongosort_list.append((f[1:], -1))
5310 else:
5311 mongosort_list.append((f, 1))
5312
5313 if limitby:
5314 limitby_skip, limitby_limit = limitby
5315 else:
5316 limitby_skip = limitby_limit = 0
5317
5318 mongofields_dict = self.SON()
5319 mongoqry_dict = {}
5320 for item in fields:
5321 if isinstance(item, SQLALL):
5322 new_fields += item._table
5323 else:
5324 new_fields.append(item)
5325 fields = new_fields
5326 if isinstance(query,Query):
5327 tablename = self.get_table(query)
5328 elif len(fields) != 0:
5329 tablename = fields[0].tablename
5330 else:
5331 raise SyntaxError("The table name could not be found in " +
5332 "the query nor from the select statement.")
5333
5334 mongoqry_dict = self.expand(query)
5335 fields = fields or self.db[tablename]
5336 for field in fields:
5337 mongofields_dict[field.name] = 1
5338
5339 return tablename, mongoqry_dict, mongofields_dict, mongosort_list, \
5340 limitby_limit, limitby_skip
5341
5342
5343 - def select(self, query, fields, attributes, count=False,
5344 snapshot=False):
5345
5346 tablename, mongoqry_dict, mongofields_dict, mongosort_list, \
5347 limitby_limit, limitby_skip = self._select(query, fields, attributes)
5348 ctable = self.connection[tablename]
5349
5350 if count:
5351 return {'count' : ctable.find(
5352 mongoqry_dict, mongofields_dict,
5353 skip=limitby_skip, limit=limitby_limit,
5354 sort=mongosort_list, snapshot=snapshot).count()}
5355 else:
5356
5357 mongo_list_dicts = ctable.find(mongoqry_dict,
5358 mongofields_dict, skip=limitby_skip,
5359 limit=limitby_limit, sort=mongosort_list,
5360 snapshot=snapshot)
5361 rows = []
5362
5363
5364 colnames = []
5365 newnames = []
5366 for field in fields:
5367 colname = str(field)
5368 colnames.append(colname)
5369 tablename, fieldname = colname.split(".")
5370 if fieldname == "_id":
5371
5372 field.name = "id"
5373 newnames.append(".".join((tablename, field.name)))
5374
5375 for record in mongo_list_dicts:
5376 row=[]
5377 for colname in colnames:
5378 tablename, fieldname = colname.split(".")
5379
5380
5381 if fieldname == "id": fieldname = "_id"
5382 if fieldname in record:
5383 if isinstance(record[fieldname],
5384 self.ObjectId):
5385 value = int(str(record[fieldname]), 16)
5386 else:
5387 value = record[fieldname]
5388 else:
5389 value = None
5390 row.append(value)
5391 rows.append(row)
5392 processor = attributes.get('processor', self.parse)
5393 result = processor(rows, fields, newnames, False)
5394 return result
5395
5396
5400
5401 - def drop(self, table, mode=''):
5404
5405
5406 - def truncate(self, table, mode, safe=None):
5407 if safe == None:
5408 safe=self.safe
5409 ctable = self.connection[table._tablename]
5410 ctable.remove(None, safe=True)
5411
5412 - def oupdate(self, tablename, query, fields):
5413 if not isinstance(query, Query):
5414 raise SyntaxError("Not Supported")
5415 filter = None
5416 if query:
5417 filter = self.expand(query)
5418 modify = {'$set': dict((k.name, self.represent(v, k.type)) for
5419 k, v in fields)}
5420 return modify, filter
5421
5422 - def update(self, tablename, query, fields, safe=None):
5423 if safe == None:
5424 safe = self.safe
5425
5426
5427 if not isinstance(query, Query):
5428 raise RuntimeError("Not implemented")
5429 amount = self.count(query, False)
5430 modify, filter = self.oupdate(tablename, query, fields)
5431 try:
5432 result = self.connection[tablename].update(filter,
5433 modify, multi=True, safe=safe)
5434 if safe:
5435 try:
5436
5437 return result["n"]
5438 except (KeyError, AttributeError, TypeError):
5439 return amount
5440 else:
5441 return amount
5442 except Exception, e:
5443
5444 raise RuntimeError("uncaught exception when updating rows: %s" % e)
5445
5446
5447 - def _update(self,tablename,query,fields):
5449
5450 - def delete(self, tablename, query, safe=None):
5451 if safe is None:
5452 safe = self.safe
5453 amount = 0
5454 amount = self.count(query, False)
5455 if not isinstance(query, Query):
5456 raise RuntimeError("query type %s is not supported" % \
5457 type(query))
5458 filter = self.expand(query)
5459 self._delete(tablename, filter, safe=safe)
5460 return amount
5461
5462 - def _delete(self, tablename, filter, safe=None):
5463 return self.connection[tablename].remove(filter, safe=safe)
5464
5466 return [self.insert(table,item) for item in items]
5467
5468
5469 - def NOT(self, first):
5470 result = {}
5471 result["$not"] = self.expand(first)
5472 return result
5473
5474 - def AND(self,first,second):
5479
5480 - def OR(self,first,second):
5481
5482 result = {}
5483 f = self.expand(first)
5484 s = self.expand(second)
5485 result['$or'] = [f,s]
5486 return result
5487
5488 - def BELONGS(self, first, second):
5489 if isinstance(second, str):
5490 return {self.expand(first) : {"$in" : [ second[:-1]]} }
5491 elif second==[] or second==():
5492 return {1:0}
5493 items = [self.expand(item, first.type) for item in second]
5494 return {self.expand(first) : {"$in" : items} }
5495
5496 - def EQ(self,first,second):
5497 result = {}
5498 result[self.expand(first)] = self.expand(second)
5499 return result
5500
5501 - def NE(self, first, second=None):
5502 result = {}
5503 result[self.expand(first)] = {'$ne': self.expand(second)}
5504 return result
5505
5506 - def LT(self,first,second=None):
5507 if second is None:
5508 raise RuntimeError("Cannot compare %s < None" % first)
5509 result = {}
5510 result[self.expand(first)] = {'$lt': self.expand(second)}
5511 return result
5512
5513 - def LE(self,first,second=None):
5514 if second is None:
5515 raise RuntimeError("Cannot compare %s <= None" % first)
5516 result = {}
5517 result[self.expand(first)] = {'$lte': self.expand(second)}
5518 return result
5519
5520 - def GT(self,first,second):
5521 result = {}
5522 result[self.expand(first)] = {'$gt': self.expand(second)}
5523 return result
5524
5525 - def GE(self,first,second=None):
5526 if second is None:
5527 raise RuntimeError("Cannot compare %s >= None" % first)
5528 result = {}
5529 result[self.expand(first)] = {'$gte': self.expand(second)}
5530 return result
5531
5532 - def ADD(self, first, second):
5536
5537 - def SUB(self, first, second):
5541
5542 - def MUL(self, first, second):
5546
5547 - def DIV(self, first, second):
5551
5552 - def MOD(self, first, second):
5556
5557 - def AS(self, first, second):
5558 raise NotImplementedError(self.error_messages["javascript_needed"])
5559 return '%s AS %s' % (self.expand(first), second)
5560
5561
5562
5563
5564 - def ON(self, first, second):
5565 raise NotImplementedError("This is not possible in NoSQL" +
5566 " but can be simulated with a wrapper.")
5567 return '%s ON %s' % (self.expand(first), self.expand(second))
5568
5569
5570
5571
5572 - def COMMA(self, first, second):
5574
5575 - def LIKE(self, first, second):
5579
5581
5582 return {self.expand(first): ('/^%s/' % \
5583 self.expand(second, 'string'))}
5584
5586
5587 return {self.expand(first): ('/%s^/' % \
5588 self.expand(second, 'string'))}
5589
5590 - def CONTAINS(self, first, second, case_sensitive=False):
5591
5592
5593
5594 return {self.expand(first) : ('/%s/' % \
5595 self.expand(second, 'string'))}
5596
5597 - def LIKE(self, first, second):
5602
5603
5605
5606 import re
5607 return {self.expand(first): {'$regex' : '^' +
5608 re.escape(self.expand(second,
5609 'string'))}}
5610
5611
5613
5614
5615
5616
5617 import re
5618 return {self.expand(first): {'$regex': \
5619 re.escape(self.expand(second, 'string')) + '$'}}
5620
5621
5622 - def CONTAINS(self, first, second, case_sensitive=False):
5623
5624
5625
5626
5627 return {self.expand(first) : {' $regex': \
5628 ".*" + re.escape(self.expand(second, 'string')) + ".*"}}
5629
5632 drivers = ('imaplib',)
5633
5634 """ IMAP server adapter
5635
5636 This class is intended as an interface with
5637 email IMAP servers to perform simple queries in the
5638 web2py DAL query syntax, so email read, search and
5639 other related IMAP mail services (as those implemented
5640 by brands like Google(r), and Yahoo!(r)
5641 can be managed from web2py applications.
5642
5643 The code uses examples by Yuji Tomita on this post:
5644 http://yuji.wordpress.com/2011/06/22/python-imaplib-imap-example-with-gmail/#comment-1137
5645 and is based in docs for Python imaplib, python email
5646 and email IETF's (i.e. RFC2060 and RFC3501)
5647
5648 This adapter was tested with a small set of operations with Gmail(r). Other
5649 services requests could raise command syntax and response data issues.
5650
5651 It creates its table and field names "statically",
5652 meaning that the developer should leave the table and field
5653 definitions to the DAL instance by calling the adapter's
5654 .define_tables() method. The tables are defined with the
5655 IMAP server mailbox list information.
5656
5657 .define_tables() returns a dictionary mapping dal tablenames
5658 to the server mailbox names with the following structure:
5659
5660 {<tablename>: str <server mailbox name>}
5661
5662 Here is a list of supported fields:
5663
5664 Field Type Description
5665 ################################################################
5666 uid string
5667 answered boolean Flag
5668 created date
5669 content list:string A list of text or html parts
5670 to string
5671 cc string
5672 bcc string
5673 size integer the amount of octets of the message*
5674 deleted boolean Flag
5675 draft boolean Flag
5676 flagged boolean Flag
5677 sender string
5678 recent boolean Flag
5679 seen boolean Flag
5680 subject string
5681 mime string The mime header declaration
5682 email string The complete RFC822 message**
5683 attachments <type list> Each non text part as dict
5684 encoding string The main detected encoding
5685
5686 *At the application side it is measured as the length of the RFC822
5687 message string
5688
5689 WARNING: As row id's are mapped to email sequence numbers,
5690 make sure your imap client web2py app does not delete messages
5691 during select or update actions, to prevent
5692 updating or deleting different messages.
5693 Sequence numbers change whenever the mailbox is updated.
5694 To avoid this sequence numbers issues, it is recommended the use
5695 of uid fields in query references (although the update and delete
5696 in separate actions rule still applies).
5697
5698 # This is the code recommended to start imap support
5699 # at the app's model:
5700
5701 imapdb = DAL("imap://user:password@server:port", pool_size=1) # port 993 for ssl
5702 imapdb.define_tables()
5703
5704 Here is an (incomplete) list of possible imap commands:
5705
5706 # Count today's unseen messages
5707 # smaller than 6000 octets from the
5708 # inbox mailbox
5709
5710 q = imapdb.INBOX.seen == False
5711 q &= imapdb.INBOX.created == datetime.date.today()
5712 q &= imapdb.INBOX.size < 6000
5713 unread = imapdb(q).count()
5714
5715 # Fetch last query messages
5716 rows = imapdb(q).select()
5717
5718 # it is also possible to filter query select results with limitby and
5719 # sequences of mailbox fields
5720
5721 set.select(<fields sequence>, limitby=(<int>, <int>))
5722
5723 # Mark last query messages as seen
5724 messages = [row.uid for row in rows]
5725 seen = imapdb(imapdb.INBOX.uid.belongs(messages)).update(seen=True)
5726
5727 # Delete messages in the imap database that have mails from mr. Gumby
5728
5729 deleted = 0
5730 for mailbox in imapdb.tables
5731 deleted += imapdb(imapdb[mailbox].sender.contains("gumby")).delete()
5732
5733 # It is possible also to mark messages for deletion instead of ereasing them
5734 # directly with set.update(deleted=True)
5735
5736
5737 # This object give access
5738 # to the adapter auto mailbox
5739 # mapped names (which native
5740 # mailbox has what table name)
5741
5742 imapdb.mailboxes <dict> # tablename, server native name pairs
5743
5744 # To retrieve a table native mailbox name use:
5745 imapdb.<table>.mailbox
5746
5747 ### New features v2.4.1:
5748
5749 # Declare mailboxes statically with tablename, name pairs
5750 # This avoids the extra server names retrieval
5751
5752 imapdb.define_tables({"inbox": "INBOX"})
5753
5754 # Selects without content/attachments/email columns will only
5755 # fetch header and flags
5756
5757 imapdb(q).select(imapdb.INBOX.sender, imapdb.INBOX.subject)
5758 """
5759
5760 types = {
5761 'string': str,
5762 'text': str,
5763 'date': datetime.date,
5764 'datetime': datetime.datetime,
5765 'id': long,
5766 'boolean': bool,
5767 'integer': int,
5768 'bigint': long,
5769 'blob': str,
5770 'list:string': str,
5771 }
5772
5773 dbengine = 'imap'
5774
5775 REGEX_URI = re.compile('^(?P<user>[^:]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?$')
5776
5777 - def __init__(self,
5778 db,
5779 uri,
5780 pool_size=0,
5781 folder=None,
5782 db_codec ='UTF-8',
5783 credential_decoder=IDENTITY,
5784 driver_args={},
5785 adapter_args={},
5786 do_connect=True,
5787 after_connection=None):
5788
5789
5790
5791
5792 self.db = db
5793 self.uri = uri
5794 if do_connect: self.find_driver(adapter_args)
5795 self.pool_size=pool_size
5796 self.folder = folder
5797 self.db_codec = db_codec
5798 self._after_connection = after_connection
5799 self.credential_decoder = credential_decoder
5800 self.driver_args = driver_args
5801 self.adapter_args = adapter_args
5802 self.mailbox_size = None
5803 self.static_names = None
5804 self.charset = sys.getfilesystemencoding()
5805
5806 self.imap4 = None
5807 uri = uri.split("://")[1]
5808
5809 """ MESSAGE is an identifier for sequence number"""
5810
5811 self.flags = ['\\Deleted', '\\Draft', '\\Flagged',
5812 '\\Recent', '\\Seen', '\\Answered']
5813 self.search_fields = {
5814 'id': 'MESSAGE', 'created': 'DATE',
5815 'uid': 'UID', 'sender': 'FROM',
5816 'to': 'TO', 'cc': 'CC',
5817 'bcc': 'BCC', 'content': 'TEXT',
5818 'size': 'SIZE', 'deleted': '\\Deleted',
5819 'draft': '\\Draft', 'flagged': '\\Flagged',
5820 'recent': '\\Recent', 'seen': '\\Seen',
5821 'subject': 'SUBJECT', 'answered': '\\Answered',
5822 'mime': None, 'email': None,
5823 'attachments': None
5824 }
5825
5826 db['_lastsql'] = ''
5827
5828 m = self.REGEX_URI.match(uri)
5829 user = m.group('user')
5830 password = m.group('password')
5831 host = m.group('host')
5832 port = int(m.group('port'))
5833 over_ssl = False
5834 if port==993:
5835 over_ssl = True
5836
5837 driver_args.update(host=host,port=port, password=password, user=user)
5838 def connector(driver_args=driver_args):
5839
5840
5841 if over_ssl:
5842 self.imap4 = self.driver.IMAP4_SSL
5843 else:
5844 self.imap4 = self.driver.IMAP4
5845 connection = self.imap4(driver_args["host"], driver_args["port"])
5846 data = connection.login(driver_args["user"], driver_args["password"])
5847
5848
5849 connection.mailbox_names = None
5850
5851
5852 connection.cursor = lambda : True
5853
5854 return connection
5855
5856 self.db.define_tables = self.define_tables
5857 self.connector = connector
5858 if do_connect: self.reconnect()
5859
5904
5906 last_message = None
5907
5908
5909 if not isinstance(self.connection.mailbox_names, dict):
5910 self.get_mailboxes()
5911 try:
5912 result = self.connection.select(self.connection.mailbox_names[tablename])
5913 last_message = int(result[1][0])
5914 except (IndexError, ValueError, TypeError, KeyError):
5915 e = sys.exc_info()[1]
5916 LOGGER.debug("Error retrieving the last mailbox sequence number. %s" % str(e))
5917 return last_message
5918
5920 if not isinstance(self.connection.mailbox_names, dict):
5921 self.get_mailboxes()
5922
5923
5924 last_message = self.get_last_message(tablename)
5925 result, data = self.connection.uid("search", None, "(ALL)")
5926 uid_list = data[0].strip().split()
5927 if len(uid_list) <= 0:
5928 return None
5929 else:
5930 return (uid_list[0], uid_list[-1])
5931
5933 if add is None:
5934 add = datetime.timedelta()
5935 """ Convert a date object to a string
5936 with d-Mon-Y style for IMAP or the inverse
5937 case
5938
5939 add <timedelta> adds to the date object
5940 """
5941 months = [None, "Jan","Feb","Mar","Apr","May","Jun",
5942 "Jul", "Aug","Sep","Oct","Nov","Dec"]
5943 if isinstance(date, basestring):
5944
5945 try:
5946 dayname, datestring = date.split(",")
5947 except (ValueError):
5948 LOGGER.debug("Could not parse date text: %s" % date)
5949 return None
5950 date_list = datestring.strip().split()
5951 year = int(date_list[2])
5952 month = months.index(date_list[1])
5953 day = int(date_list[0])
5954 hms = map(int, date_list[3].split(":"))
5955 return datetime.datetime(year, month, day,
5956 hms[0], hms[1], hms[2]) + add
5957 elif isinstance(date, (datetime.datetime, datetime.date)):
5958 return (date + add).strftime("%d-%b-%Y")
5959
5960 else:
5961 return None
5962
5963 @staticmethod
5965 from email.header import decode_header
5966 text, encoding = decode_header(f)[0]
5967 return text
5968
5969 - def encode_text(self, text, charset, errors="replace"):
5970 """ convert text for mail to unicode"""
5971 if text is None:
5972 text = ""
5973 else:
5974 if isinstance(text, str):
5975 if charset is None:
5976 text = unicode(text, "utf-8", errors)
5977 else:
5978 text = unicode(text, charset, errors)
5979 else:
5980 raise Exception("Unsupported mail text type %s" % type(text))
5981 return text.encode("utf-8")
5982
5984 charset = message.get_content_charset()
5985 return charset
5986
5988 """ Query the mail database for mailbox names """
5989 if self.static_names:
5990
5991 self.connection.mailbox_names = self.static_names
5992 return self.static_names.keys()
5993
5994 mailboxes_list = self.connection.list()
5995 self.connection.mailbox_names = dict()
5996 mailboxes = list()
5997 x = 0
5998 for item in mailboxes_list[1]:
5999 x = x + 1
6000 item = item.strip()
6001 if not "NOSELECT" in item.upper():
6002 sub_items = item.split("\"")
6003 sub_items = [sub_item for sub_item in sub_items \
6004 if len(sub_item.strip()) > 0]
6005
6006 mailbox = sub_items[-1]
6007
6008
6009 mailbox_name = re.sub('^[_0-9]*', '', re.sub('[^_\w]','',re.sub('[/ ]','_',mailbox)))
6010 mailboxes.append(mailbox_name)
6011 self.connection.mailbox_names[mailbox_name] = mailbox
6012
6013 return mailboxes
6014
6016 nofield = True
6017 tablename = None
6018 attr = query
6019 while nofield:
6020 if hasattr(attr, "first"):
6021 attr = attr.first
6022 if isinstance(attr, Field):
6023 return attr.tablename
6024 elif isinstance(attr, Query):
6025 pass
6026 else:
6027 return None
6028 else:
6029 return None
6030 return tablename
6031
6033 if self.search_fields.get(flag, None) in self.flags:
6034 return True
6035 else:
6036 return False
6037
6039 """
6040 Auto create common IMAP fileds
6041
6042 This function creates fields definitions "statically"
6043 meaning that custom fields as in other adapters should
6044 not be supported and definitions handled on a service/mode
6045 basis (local syntax for Gmail(r), Ymail(r)
6046
6047 Returns a dictionary with tablename, server native mailbox name
6048 pairs.
6049 """
6050 if mailbox_names:
6051
6052 self.static_names = mailbox_names
6053 else:
6054 self.static_names = None
6055 if not isinstance(self.connection.mailbox_names, dict):
6056 self.get_mailboxes()
6057
6058 names = self.connection.mailbox_names.keys()
6059
6060 for name in names:
6061 self.db.define_table("%s" % name,
6062 Field("uid", "string", writable=False),
6063 Field("answered", "boolean"),
6064 Field("created", "datetime", writable=False),
6065 Field("content", "list:string", writable=False),
6066 Field("to", "string", writable=False),
6067 Field("cc", "string", writable=False),
6068 Field("bcc", "string", writable=False),
6069 Field("size", "integer", writable=False),
6070 Field("deleted", "boolean"),
6071 Field("draft", "boolean"),
6072 Field("flagged", "boolean"),
6073 Field("sender", "string", writable=False),
6074 Field("recent", "boolean", writable=False),
6075 Field("seen", "boolean"),
6076 Field("subject", "string", writable=False),
6077 Field("mime", "string", writable=False),
6078 Field("email", "string", writable=False, readable=False),
6079 Field("attachments", list, writable=False, readable=False),
6080 Field("encoding")
6081 )
6082
6083
6084
6085 self.db[name].mailbox = \
6086 self.connection.mailbox_names[name]
6087
6088
6089 self.db[name].to.represent = self.db[name].cc.represent = \
6090 self.db[name].bcc.represent = self.db[name].sender.represent = \
6091 self.db[name].subject.represent = self.header_represent
6092
6093
6094 self.db.mailboxes = self.connection.mailbox_names
6095 return self.db.mailboxes
6096
6101
6102 - def _select(self, query, fields, attributes):
6106
6107 - def select(self, query, fields, attributes):
6108 """ Search and Fetch records and return web2py rows
6109 """
6110
6111 if use_common_filters(query):
6112 query = self.common_filter(query, [self.get_query_mailbox(query),])
6113
6114 import email
6115
6116
6117 tablename = None
6118 fetch_results = list()
6119
6120 if isinstance(query, Query):
6121 tablename = self.get_table(query)
6122 mailbox = self.connection.mailbox_names.get(tablename, None)
6123 if mailbox is None:
6124 raise ValueError("Mailbox name not found: %s" % mailbox)
6125 else:
6126
6127 result, selected = self.connection.select(mailbox, True)
6128 if result != "OK":
6129 raise Exception("IMAP error: %s" % selected)
6130 self.mailbox_size = int(selected[0])
6131 search_query = "(%s)" % str(query).strip()
6132 search_result = self.connection.uid("search", None, search_query)
6133
6134 if search_result[0] == "OK":
6135
6136
6137
6138
6139 limitby = attributes.get('limitby', None)
6140 messages_set = search_result[1][0].split()
6141
6142 messages_set.reverse()
6143 if limitby is not None:
6144
6145 messages_set = messages_set[int(limitby[0]):int(limitby[1])]
6146
6147
6148 if any([(field.name in ["content", "size",
6149 "attachments", "email"]) for
6150 field in fields]):
6151 imap_fields = "(RFC822 FLAGS)"
6152 else:
6153 imap_fields = "(RFC822.HEADER FLAGS)"
6154
6155 if len(messages_set) > 0:
6156
6157
6158
6159
6160 for uid in messages_set:
6161
6162 typ, data = self.connection.uid("fetch", uid, imap_fields)
6163 if typ == "OK":
6164 fr = {"message": int(data[0][0].split()[0]),
6165 "uid": int(uid),
6166 "email": email.message_from_string(data[0][1]),
6167 "raw_message": data[0][1]}
6168 fr["multipart"] = fr["email"].is_multipart()
6169
6170 fr["flags"] = self.driver.ParseFlags(data[1])
6171 fetch_results.append(fr)
6172 else:
6173
6174 raise Exception("IMAP error retrieving the body: %s" % data)
6175 else:
6176 raise Exception("IMAP search error: %s" % search_result[1])
6177 elif isinstance(query, (Expression, basestring)):
6178 raise NotImplementedError()
6179 else:
6180 raise TypeError("Unexpected query type")
6181
6182 imapqry_dict = {}
6183 imapfields_dict = {}
6184
6185 if len(fields) == 1 and isinstance(fields[0], SQLALL):
6186 allfields = True
6187 elif len(fields) == 0:
6188 allfields = True
6189 else:
6190 allfields = False
6191 if allfields:
6192 colnames = ["%s.%s" % (tablename, field) for field in self.search_fields.keys()]
6193 else:
6194 colnames = ["%s.%s" % (tablename, field.name) for field in fields]
6195
6196 for k in colnames:
6197 imapfields_dict[k] = k
6198
6199 imapqry_list = list()
6200 imapqry_array = list()
6201 for fr in fetch_results:
6202 attachments = []
6203 content = []
6204 size = 0
6205 n = int(fr["message"])
6206 item_dict = dict()
6207 message = fr["email"]
6208 uid = fr["uid"]
6209 charset = self.get_charset(message)
6210 flags = fr["flags"]
6211 raw_message = fr["raw_message"]
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222 if "%s.id" % tablename in colnames:
6223 item_dict["%s.id" % tablename] = n
6224 if "%s.created" % tablename in colnames:
6225 item_dict["%s.created" % tablename] = self.convert_date(message["Date"])
6226 if "%s.uid" % tablename in colnames:
6227 item_dict["%s.uid" % tablename] = uid
6228 if "%s.sender" % tablename in colnames:
6229
6230
6231
6232 item_dict["%s.sender" % tablename] = message["From"]
6233 if "%s.to" % tablename in colnames:
6234 item_dict["%s.to" % tablename] = message["To"]
6235 if "%s.cc" % tablename in colnames:
6236 if "Cc" in message.keys():
6237 item_dict["%s.cc" % tablename] = message["Cc"]
6238 else:
6239 item_dict["%s.cc" % tablename] = ""
6240 if "%s.bcc" % tablename in colnames:
6241 if "Bcc" in message.keys():
6242 item_dict["%s.bcc" % tablename] = message["Bcc"]
6243 else:
6244 item_dict["%s.bcc" % tablename] = ""
6245 if "%s.deleted" % tablename in colnames:
6246 item_dict["%s.deleted" % tablename] = "\\Deleted" in flags
6247 if "%s.draft" % tablename in colnames:
6248 item_dict["%s.draft" % tablename] = "\\Draft" in flags
6249 if "%s.flagged" % tablename in colnames:
6250 item_dict["%s.flagged" % tablename] = "\\Flagged" in flags
6251 if "%s.recent" % tablename in colnames:
6252 item_dict["%s.recent" % tablename] = "\\Recent" in flags
6253 if "%s.seen" % tablename in colnames:
6254 item_dict["%s.seen" % tablename] = "\\Seen" in flags
6255 if "%s.subject" % tablename in colnames:
6256 item_dict["%s.subject" % tablename] = message["Subject"]
6257 if "%s.answered" % tablename in colnames:
6258 item_dict["%s.answered" % tablename] = "\\Answered" in flags
6259 if "%s.mime" % tablename in colnames:
6260 item_dict["%s.mime" % tablename] = message.get_content_type()
6261 if "%s.encoding" % tablename in colnames:
6262 item_dict["%s.encoding" % tablename] = charset
6263
6264
6265
6266
6267
6268
6269 if "%s.email" % tablename in colnames:
6270
6271 item_dict["%s.email" % tablename] = raw_message
6272
6273
6274
6275
6276
6277
6278 for part in message.walk():
6279 maintype = part.get_content_maintype()
6280 if ("%s.attachments" % tablename in colnames) or \
6281 ("%s.content" % tablename in colnames):
6282 if "%s.attachments" % tablename in colnames:
6283 if not ("text" in maintype):
6284 payload = part.get_payload(decode=True)
6285 if payload:
6286 attachment = {
6287 "payload": payload,
6288 "filename": part.get_filename(),
6289 "encoding": part.get_content_charset(),
6290 "mime": part.get_content_type(),
6291 "disposition": part["Content-Disposition"]}
6292 attachments.append(attachment)
6293 if "%s.content" % tablename in colnames:
6294 payload = part.get_payload(decode=True)
6295 part_charset = self.get_charset(part)
6296 if "text" in maintype:
6297 if payload:
6298 content.append(self.encode_text(payload, part_charset))
6299 if "%s.size" % tablename in colnames:
6300 if part is not None:
6301 size += len(str(part))
6302 item_dict["%s.content" % tablename] = bar_encode(content)
6303 item_dict["%s.attachments" % tablename] = attachments
6304 item_dict["%s.size" % tablename] = size
6305 imapqry_list.append(item_dict)
6306
6307
6308
6309 for item_dict in imapqry_list:
6310 imapqry_array_item = list()
6311 for fieldname in colnames:
6312 imapqry_array_item.append(item_dict[fieldname])
6313 imapqry_array.append(imapqry_array_item)
6314
6315
6316 colnames = colnames
6317 processor = attributes.get('processor',self.parse)
6318 return processor(imapqry_array, fields, colnames)
6319
6320 - def _update(self, tablename, query, fields, commit=False):
6321
6322 commands = list()
6323 if use_common_filters(query):
6324 query = self.common_filter(query, [tablename,])
6325 mark = []
6326 unmark = []
6327 if query:
6328 for item in fields:
6329 field = item[0]
6330 name = field.name
6331 value = item[1]
6332 if self.is_flag(name):
6333 flag = self.search_fields[name]
6334 if (value is not None) and (flag != "\\Recent"):
6335 if value:
6336 mark.append(flag)
6337 else:
6338 unmark.append(flag)
6339 result, data = self.connection.select(
6340 self.connection.mailbox_names[tablename])
6341 string_query = "(%s)" % query
6342 result, data = self.connection.search(None, string_query)
6343 store_list = [item.strip() for item in data[0].split()
6344 if item.strip().isdigit()]
6345
6346 for number in store_list:
6347 result = None
6348 if len(mark) > 0:
6349 commands.append((number, "+FLAGS", "(%s)" % " ".join(mark)))
6350 if len(unmark) > 0:
6351 commands.append((number, "-FLAGS", "(%s)" % " ".join(unmark)))
6352 return commands
6353
6354 - def update(self, tablename, query, fields):
6355 rowcount = 0
6356 commands = self._update(tablename, query, fields)
6357 for command in commands:
6358 result, data = self.connection.store(*command)
6359 if result == "OK":
6360 rowcount += 1
6361 else:
6362 raise Exception("IMAP storing error: %s" % data)
6363 return rowcount
6364
6365 - def _count(self, query, distinct=None):
6366 raise NotImplementedError()
6367
6368 - def count(self,query,distinct=None):
6380
6381 - def delete(self, tablename, query):
6382 counter = 0
6383 if query:
6384 if use_common_filters(query):
6385 query = self.common_filter(query, [tablename,])
6386 result, data = self.connection.select(self.connection.mailbox_names[tablename])
6387 string_query = "(%s)" % query
6388 result, data = self.connection.search(None, string_query)
6389 store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()]
6390 for number in store_list:
6391 result, data = self.connection.store(number, "+FLAGS", "(\\Deleted)")
6392 if result == "OK":
6393 counter += 1
6394 else:
6395 raise Exception("IMAP store error: %s" % data)
6396 if counter > 0:
6397 result, data = self.connection.expunge()
6398 return counter
6399
6400 - def BELONGS(self, first, second):
6401 result = None
6402 name = self.search_fields[first.name]
6403 if name == "MESSAGE":
6404 values = [str(val) for val in second if str(val).isdigit()]
6405 result = "%s" % ",".join(values).strip()
6406
6407 elif name == "UID":
6408 values = [str(val) for val in second if str(val).isdigit()]
6409 result = "UID %s" % ",".join(values).strip()
6410
6411 else:
6412 raise Exception("Operation not supported")
6413
6414 return result
6415
6416 - def CONTAINS(self, first, second, case_sensitive=False):
6417
6418 result = None
6419 name = self.search_fields[first.name]
6420
6421 if name in ("FROM", "TO", "SUBJECT", "TEXT"):
6422 result = "%s \"%s\"" % (name, self.expand(second))
6423 else:
6424 if first.name in ("cc", "bcc"):
6425 result = "%s \"%s\"" % (first.name.upper(), self.expand(second))
6426 elif first.name == "mime":
6427 result = "HEADER Content-Type \"%s\"" % self.expand(second)
6428 else:
6429 raise Exception("Operation not supported")
6430 return result
6431
6432 - def GT(self, first, second):
6433 result = None
6434 name = self.search_fields[first.name]
6435 if name == "MESSAGE":
6436 last_message = self.get_last_message(first.tablename)
6437 result = "%d:%d" % (int(self.expand(second)) + 1, last_message)
6438 elif name == "UID":
6439
6440
6441
6442 try:
6443 pedestal, threshold = self.get_uid_bounds(first.tablename)
6444 except TypeError:
6445 e = sys.exc_info()[1]
6446 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6447 return ""
6448 try:
6449 lower_limit = int(self.expand(second)) + 1
6450 except (ValueError, TypeError):
6451 e = sys.exc_info()[1]
6452 raise Exception("Operation not supported (non integer UID)")
6453 result = "UID %s:%s" % (lower_limit, threshold)
6454 elif name == "DATE":
6455 result = "SINCE %s" % self.convert_date(second, add=datetime.timedelta(1))
6456 elif name == "SIZE":
6457 result = "LARGER %s" % self.expand(second)
6458 else:
6459 raise Exception("Operation not supported")
6460 return result
6461
6462 - def GE(self, first, second):
6463 result = None
6464 name = self.search_fields[first.name]
6465 if name == "MESSAGE":
6466 last_message = self.get_last_message(first.tablename)
6467 result = "%s:%s" % (self.expand(second), last_message)
6468 elif name == "UID":
6469
6470
6471
6472 try:
6473 pedestal, threshold = self.get_uid_bounds(first.tablename)
6474 except TypeError:
6475 e = sys.exc_info()[1]
6476 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6477 return ""
6478 lower_limit = self.expand(second)
6479 result = "UID %s:%s" % (lower_limit, threshold)
6480 elif name == "DATE":
6481 result = "SINCE %s" % self.convert_date(second)
6482 else:
6483 raise Exception("Operation not supported")
6484 return result
6485
6486 - def LT(self, first, second):
6487 result = None
6488 name = self.search_fields[first.name]
6489 if name == "MESSAGE":
6490 result = "%s:%s" % (1, int(self.expand(second)) - 1)
6491 elif name == "UID":
6492 try:
6493 pedestal, threshold = self.get_uid_bounds(first.tablename)
6494 except TypeError:
6495 e = sys.exc_info()[1]
6496 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6497 return ""
6498 try:
6499 upper_limit = int(self.expand(second)) - 1
6500 except (ValueError, TypeError):
6501 e = sys.exc_info()[1]
6502 raise Exception("Operation not supported (non integer UID)")
6503 result = "UID %s:%s" % (pedestal, upper_limit)
6504 elif name == "DATE":
6505 result = "BEFORE %s" % self.convert_date(second)
6506 elif name == "SIZE":
6507 result = "SMALLER %s" % self.expand(second)
6508 else:
6509 raise Exception("Operation not supported")
6510 return result
6511
6512 - def LE(self, first, second):
6513 result = None
6514 name = self.search_fields[first.name]
6515 if name == "MESSAGE":
6516 result = "%s:%s" % (1, self.expand(second))
6517 elif name == "UID":
6518 try:
6519 pedestal, threshold = self.get_uid_bounds(first.tablename)
6520 except TypeError:
6521 e = sys.exc_info()[1]
6522 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6523 return ""
6524 upper_limit = int(self.expand(second))
6525 result = "UID %s:%s" % (pedestal, upper_limit)
6526 elif name == "DATE":
6527 result = "BEFORE %s" % self.convert_date(second, add=datetime.timedelta(1))
6528 else:
6529 raise Exception("Operation not supported")
6530 return result
6531
6532 - def NE(self, first, second=None):
6533 if (second is None) and isinstance(first, Field):
6534
6535 if first.type == "id":
6536 return self.GE(first, 1)
6537 result = self.NOT(self.EQ(first, second))
6538 result = result.replace("NOT NOT", "").strip()
6539 return result
6540
6541 - def EQ(self,first,second):
6542 name = self.search_fields[first.name]
6543 result = None
6544 if name is not None:
6545 if name == "MESSAGE":
6546
6547 result = "%s" % self.expand(second)
6548 elif name == "UID":
6549 result = "UID %s" % self.expand(second)
6550 elif name == "DATE":
6551 result = "ON %s" % self.convert_date(second)
6552
6553 elif name in self.flags:
6554 if second:
6555 result = "%s" % (name.upper()[1:])
6556 else:
6557 result = "NOT %s" % (name.upper()[1:])
6558 else:
6559 raise Exception("Operation not supported")
6560 else:
6561 raise Exception("Operation not supported")
6562 return result
6563
6564 - def AND(self, first, second):
6567
6568 - def OR(self, first, second):
6571
6572 - def NOT(self, first):
6573 result = "NOT %s" % self.expand(first)
6574 return result
6575
6576
6577
6578
6579
6580 ADAPTERS = {
6581 'sqlite': SQLiteAdapter,
6582 'spatialite': SpatiaLiteAdapter,
6583 'sqlite:memory': SQLiteAdapter,
6584 'spatialite:memory': SpatiaLiteAdapter,
6585 'mysql': MySQLAdapter,
6586 'postgres': PostgreSQLAdapter,
6587 'postgres:psycopg2': PostgreSQLAdapter,
6588 'postgres:pg8000': PostgreSQLAdapter,
6589 'postgres2:psycopg2': NewPostgreSQLAdapter,
6590 'postgres2:pg8000': NewPostgreSQLAdapter,
6591 'oracle': OracleAdapter,
6592 'mssql': MSSQLAdapter,
6593 'mssql2': MSSQL2Adapter,
6594 'mssql3': MSSQL3Adapter,
6595 'sybase': SybaseAdapter,
6596 'db2': DB2Adapter,
6597 'teradata': TeradataAdapter,
6598 'informix': InformixAdapter,
6599 'informix-se': InformixSEAdapter,
6600 'firebird': FireBirdAdapter,
6601 'firebird_embedded': FireBirdAdapter,
6602 'ingres': IngresAdapter,
6603 'ingresu': IngresUnicodeAdapter,
6604 'sapdb': SAPDBAdapter,
6605 'cubrid': CubridAdapter,
6606 'jdbc:sqlite': JDBCSQLiteAdapter,
6607 'jdbc:sqlite:memory': JDBCSQLiteAdapter,
6608 'jdbc:postgres': JDBCPostgreSQLAdapter,
6609 'gae': GoogleDatastoreAdapter,
6610 'google:datastore': GoogleDatastoreAdapter,
6611 'google:sql': GoogleSQLAdapter,
6612 'couchdb': CouchDBAdapter,
6613 'mongodb': MongoDBAdapter,
6614 'imap': IMAPAdapter
6615 }
6619 """
6620 Field type validation, using web2py's validators mechanism.
6621
6622 makes sure the content of a field is in line with the declared
6623 fieldtype
6624 """
6625 db = field.db
6626 if not have_validators:
6627 return []
6628 field_type, field_length = field.type, field.length
6629 if isinstance(field_type, SQLCustomType):
6630 if hasattr(field_type, 'validator'):
6631 return field_type.validator
6632 else:
6633 field_type = field_type.type
6634 elif not isinstance(field_type,str):
6635 return []
6636 requires=[]
6637 def ff(r,id):
6638 row=r(id)
6639 if not row:
6640 return id
6641 elif hasattr(r, '_format') and isinstance(r._format,str):
6642 return r._format % row
6643 elif hasattr(r, '_format') and callable(r._format):
6644 return r._format(row)
6645 else:
6646 return id
6647 if field_type in (('string', 'text', 'password')):
6648 requires.append(validators.IS_LENGTH(field_length))
6649 elif field_type == 'json':
6650 requires.append(validators.IS_EMPTY_OR(validators.IS_JSON()))
6651 elif field_type == 'double' or field_type == 'float':
6652 requires.append(validators.IS_FLOAT_IN_RANGE(-1e100, 1e100))
6653 elif field_type in ('integer','bigint'):
6654 requires.append(validators.IS_INT_IN_RANGE(-1e100, 1e100))
6655 elif field_type.startswith('decimal'):
6656 requires.append(validators.IS_DECIMAL_IN_RANGE(-10**10, 10**10))
6657 elif field_type == 'date':
6658 requires.append(validators.IS_DATE())
6659 elif field_type == 'time':
6660 requires.append(validators.IS_TIME())
6661 elif field_type == 'datetime':
6662 requires.append(validators.IS_DATETIME())
6663 elif db and field_type.startswith('reference') and \
6664 field_type.find('.') < 0 and \
6665 field_type[10:] in db.tables:
6666 referenced = db[field_type[10:]]
6667 def repr_ref(id, row=None, r=referenced, f=ff): return f(r, id)
6668 field.represent = field.represent or repr_ref
6669 if hasattr(referenced, '_format') and referenced._format:
6670 requires = validators.IS_IN_DB(db,referenced._id,
6671 referenced._format)
6672 if field.unique:
6673 requires._and = validators.IS_NOT_IN_DB(db,field)
6674 if field.tablename == field_type[10:]:
6675 return validators.IS_EMPTY_OR(requires)
6676 return requires
6677 elif db and field_type.startswith('list:reference') and \
6678 field_type.find('.') < 0 and \
6679 field_type[15:] in db.tables:
6680 referenced = db[field_type[15:]]
6681 def list_ref_repr(ids, row=None, r=referenced, f=ff):
6682 if not ids:
6683 return None
6684 refs = None
6685 db, id = r._db, r._id
6686 if isinstance(db._adapter, GoogleDatastoreAdapter):
6687 def count(values): return db(id.belongs(values)).select(id)
6688 rx = range(0, len(ids), 30)
6689 refs = reduce(lambda a,b:a&b, [count(ids[i:i+30]) for i in rx])
6690 else:
6691 refs = db(id.belongs(ids)).select(id)
6692 return (refs and ', '.join(str(f(r,x.id)) for x in refs) or '')
6693 field.represent = field.represent or list_ref_repr
6694 if hasattr(referenced, '_format') and referenced._format:
6695 requires = validators.IS_IN_DB(db,referenced._id,
6696 referenced._format,multiple=True)
6697 else:
6698 requires = validators.IS_IN_DB(db,referenced._id,
6699 multiple=True)
6700 if field.unique:
6701 requires._and = validators.IS_NOT_IN_DB(db,field)
6702 return requires
6703 elif field_type.startswith('list:'):
6704 def repr_list(values,row=None): return', '.join(str(v) for v in (values or []))
6705 field.represent = field.represent or repr_list
6706 if field.unique:
6707 requires.insert(0,validators.IS_NOT_IN_DB(db,field))
6708 sff = ['in', 'do', 'da', 'ti', 'de', 'bo']
6709 if field.notnull and not field_type[:2] in sff:
6710 requires.insert(0, validators.IS_NOT_EMPTY())
6711 elif not field.notnull and field_type[:2] in sff and requires:
6712 requires[-1] = validators.IS_EMPTY_OR(requires[-1])
6713 return requires
6714
6717 return str(item).replace('|', '||')
6718
6721
6723 if not hasattr(value,'split') and hasattr(value,'read'):
6724 value = value.read()
6725 return [int(x) for x in value.split('|') if x.strip()]
6726
6730
6731
6732 -class Row(object):
6733
6734 """
6735 a dictionary that lets you do d['a'] as well as d.a
6736 this is only used to store a Row
6737 """
6738
6740 self.__dict__.update(*args,**kwargs)
6741
6743 key=str(key)
6744 m = REGEX_TABLE_DOT_FIELD.match(key)
6745 if key in self.get('_extra',{}):
6746 return self._extra[key]
6747 elif m:
6748 try:
6749 return ogetattr(self, m.group(1))[m.group(2)]
6750 except (KeyError,AttributeError,TypeError):
6751 key = m.group(2)
6752 return ogetattr(self, key)
6753
6755 setattr(self, str(key), value)
6756
6757 __delitem__ = delattr
6758
6759 __copy__ = lambda self: Row(self)
6760
6761 __call__ = __getitem__
6762
6763 - def get(self,key,default=None):
6764 return self.__dict__.get(key,default)
6765
6767 return key in self.__dict__
6768
6769 has_key = __contains__
6770
6772 return len(self.__dict__)>0
6773
6774 - def update(self, *args, **kwargs):
6775 self.__dict__.update(*args, **kwargs)
6776
6778 return self.__dict__.keys()
6779
6781 return self.__dict__.items()
6782
6784 return self.__dict__.values()
6785
6788
6791
6793
6794 return '<Row %s>' % self.as_dict()
6795
6797 return '<Row %s>' % self.as_dict()
6798
6800 return object.__getattribute__(self,'id')
6801
6803 try:
6804 return self.as_dict() == other.as_dict()
6805 except AttributeError:
6806 return False
6807
6809 return not (self == other)
6810
6812 return Row(dict(self))
6813
6814 - def as_dict(self, datetime_to_str=False, custom_types=None):
6815 SERIALIZABLE_TYPES = [str, unicode, int, long, float, bool, list, dict]
6816 if isinstance(custom_types,(list,tuple,set)):
6817 SERIALIZABLE_TYPES += custom_types
6818 elif custom_types:
6819 SERIALIZABLE_TYPES.append(custom_types)
6820 d = dict(self)
6821 for k in copy.copy(d.keys()):
6822 v=d[k]
6823 if d[k] is None:
6824 continue
6825 elif isinstance(v,Row):
6826 d[k]=v.as_dict()
6827 elif isinstance(v,Reference):
6828 d[k]=int(v)
6829 elif isinstance(v,decimal.Decimal):
6830 d[k]=float(v)
6831 elif isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
6832 if datetime_to_str:
6833 d[k] = v.isoformat().replace('T',' ')[:19]
6834 elif not isinstance(v,tuple(SERIALIZABLE_TYPES)):
6835 del d[k]
6836 return d
6837
6838 - def as_xml(self, row_name="row", colnames=None, indent=' '):
6839 def f(row,field,indent=' '):
6840 if isinstance(row,Row):
6841 spc = indent+' \n'
6842 items = [f(row[x],x,indent+' ') for x in row]
6843 return '%s<%s>\n%s\n%s</%s>' % (
6844 indent,
6845 field,
6846 spc.join(item for item in items if item),
6847 indent,
6848 field)
6849 elif not callable(row):
6850 if REGEX_ALPHANUMERIC.match(field):
6851 return '%s<%s>%s</%s>' % (indent,field,row,field)
6852 else:
6853 return '%s<extra name="%s">%s</extra>' % \
6854 (indent,field,row)
6855 else:
6856 return None
6857 return f(self, row_name, indent=indent)
6858
6859 - def as_json(self, mode="object", default=None, colnames=None,
6860 serialize=True, **kwargs):
6861 """
6862 serializes the table to a JSON list of objects
6863 kwargs are passed to .as_dict method
6864 only "object" mode supported for single row
6865
6866 serialize = False used by Rows.as_json
6867 TODO: return array mode with query column order
6868 """
6869
6870 def inner_loop(record, col):
6871 (t, f) = col.split('.')
6872 res = None
6873 if not REGEX_TABLE_DOT_FIELD.match(col):
6874 key = col
6875 res = record._extra[col]
6876 else:
6877 key = f
6878 if isinstance(record.get(t, None), Row):
6879 res = record[t][f]
6880 else:
6881 res = record[f]
6882 if mode == 'object':
6883 return (key, res)
6884 else:
6885 return res
6886
6887 multi = any([isinstance(v, self.__class__) for v in self.values()])
6888 mode = mode.lower()
6889 if not mode in ['object', 'array']:
6890 raise SyntaxError('Invalid JSON serialization mode: %s' % mode)
6891
6892 if mode=='object' and colnames:
6893 item = dict([inner_loop(self, col) for col in colnames])
6894 elif colnames:
6895 item = [inner_loop(self, col) for col in colnames]
6896 else:
6897 if not mode == 'object':
6898 raise SyntaxError('Invalid JSON serialization mode: %s' % mode)
6899
6900 if multi:
6901 item = dict()
6902 [item.update(**v.as_dict(**kwargs)) for v in self.values()]
6903 else:
6904 item = self.as_dict(**kwargs)
6905
6906 if serialize:
6907 if have_serializers:
6908 return serializers.json(item,
6909 default=default or
6910 serializers.custom_json)
6911 elif simplejson:
6912 return simplejson.dumps(item)
6913 else:
6914 raise RuntimeError("missing simplejson")
6915 else:
6916 return item
6917
6927
6929 if not isinstance(fields,(list,tuple)):
6930 fields = [fields]
6931 new_fields = []
6932 for field in fields:
6933 if isinstance(field,Field):
6934 new_fields.append(field)
6935 elif isinstance(field,Table):
6936 for ofield in field:
6937 new_fields.append(ofield)
6938 else:
6939 raise RuntimeError("fields must be a list of fields")
6940 fields = new_fields
6941 field_map = {}
6942 for field in fields:
6943 n = field.name.lower()
6944 if not n in field_map:
6945 field_map[n] = field
6946 n = str(field).lower()
6947 if not n in field_map:
6948 field_map[n] = field
6949 constants = {}
6950 i = 0
6951 while True:
6952 m = REGEX_CONST_STRING.search(text)
6953 if not m: break
6954 text = text[:m.start()]+('#%i' % i)+text[m.end():]
6955 constants[str(i)] = m.group()[1:-1]
6956 i+=1
6957 text = re.sub('\s+',' ',text).lower()
6958 for a,b in [('&','and'),
6959 ('|','or'),
6960 ('~','not'),
6961 ('==','='),
6962 ('<','<'),
6963 ('>','>'),
6964 ('<=','<='),
6965 ('>=','>='),
6966 ('<>','!='),
6967 ('=<','<='),
6968 ('=>','>='),
6969 ('=','='),
6970 (' less or equal than ','<='),
6971 (' greater or equal than ','>='),
6972 (' equal or less than ','<='),
6973 (' equal or greater than ','>='),
6974 (' less or equal ','<='),
6975 (' greater or equal ','>='),
6976 (' equal or less ','<='),
6977 (' equal or greater ','>='),
6978 (' not equal to ','!='),
6979 (' not equal ','!='),
6980 (' equal to ','='),
6981 (' equal ','='),
6982 (' equals ','='),
6983 (' less than ','<'),
6984 (' greater than ','>'),
6985 (' starts with ','startswith'),
6986 (' ends with ','endswith'),
6987 (' not in ' , 'notbelongs'),
6988 (' in ' , 'belongs'),
6989 (' is ','=')]:
6990 if a[0]==' ':
6991 text = text.replace(' is'+a,' %s ' % b)
6992 text = text.replace(a,' %s ' % b)
6993 text = re.sub('\s+',' ',text).lower()
6994 text = re.sub('(?P<a>[\<\>\!\=])\s+(?P<b>[\<\>\!\=])','\g<a>\g<b>',text)
6995 query = field = neg = op = logic = None
6996 for item in text.split():
6997 if field is None:
6998 if item == 'not':
6999 neg = True
7000 elif not neg and not logic and item in ('and','or'):
7001 logic = item
7002 elif item in field_map:
7003 field = field_map[item]
7004 else:
7005 raise RuntimeError("Invalid syntax")
7006 elif not field is None and op is None:
7007 op = item
7008 elif not op is None:
7009 if item.startswith('#'):
7010 if not item[1:] in constants:
7011 raise RuntimeError("Invalid syntax")
7012 value = constants[item[1:]]
7013 else:
7014 value = item
7015 if field.type in ('text', 'string', 'json'):
7016 if op == '=': op = 'like'
7017 if op == '=': new_query = field==value
7018 elif op == '<': new_query = field<value
7019 elif op == '>': new_query = field>value
7020 elif op == '<=': new_query = field<=value
7021 elif op == '>=': new_query = field>=value
7022 elif op == '!=': new_query = field!=value
7023 elif op == 'belongs': new_query = field.belongs(value.split(','))
7024 elif op == 'notbelongs': new_query = ~field.belongs(value.split(','))
7025 elif field.type in ('text', 'string', 'json'):
7026 if op == 'contains': new_query = field.contains(value)
7027 elif op == 'like': new_query = field.like(value)
7028 elif op == 'startswith': new_query = field.startswith(value)
7029 elif op == 'endswith': new_query = field.endswith(value)
7030 else: raise RuntimeError("Invalid operation")
7031 elif field._db._adapter.dbengine=='google:datastore' and \
7032 field.type in ('list:integer', 'list:string', 'list:reference'):
7033 if op == 'contains': new_query = field.contains(value)
7034 else: raise RuntimeError("Invalid operation")
7035 else: raise RuntimeError("Invalid operation")
7036 if neg: new_query = ~new_query
7037 if query is None:
7038 query = new_query
7039 elif logic == 'and':
7040 query &= new_query
7041 elif logic == 'or':
7042 query |= new_query
7043 field = op = neg = logic = None
7044 return query
7045
7047
7048 """
7049 an instance of this class represents a database connection
7050
7051 Example::
7052
7053 db = DAL('sqlite://test.db')
7054 db.define_table('tablename', Field('fieldname1'),
7055 Field('fieldname2'))
7056
7057 (experimental)
7058 you can pass a dict object as uri with the uri string
7059 and table/field definitions. For an example of valid data check
7060 the output of:
7061
7062 >>> db.as_dict(flat=True, sanitize=False)
7063 """
7064
7065 - def __new__(cls, uri='sqlite://dummy.db', *args, **kwargs):
7092
7093 @staticmethod
7095 """
7096 # ## this allows gluon to set a folder for this thread
7097 # ## <<<<<<<<< Should go away as new DAL replaces old sql.py
7098 """
7099 BaseAdapter.set_folder(folder)
7100
7101 @staticmethod
7103 """
7104 Returns a dictionary with uri as key with timings and defined tables
7105 {'sqlite://storage.sqlite': {
7106 'dbstats': [(select auth_user.email from auth_user, 0.02009)],
7107 'dbtables': {
7108 'defined': ['auth_cas', 'auth_event', 'auth_group',
7109 'auth_membership', 'auth_permission', 'auth_user'],
7110 'lazy': '[]'
7111 }
7112 }
7113 }
7114 """
7115 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
7116 infos = {}
7117 for db_uid, db_group in dbs:
7118 for db in db_group:
7119 if not db._uri:
7120 continue
7121 k = hide_password(db._uri)
7122 infos[k] = dict(dbstats = [(row[0], row[1]) for row in db._timings],
7123 dbtables = {'defined':
7124 sorted(list(set(db.tables) -
7125 set(db._LAZY_TABLES.keys()))),
7126 'lazy': sorted(db._LAZY_TABLES.keys())}
7127 )
7128 return infos
7129
7130 @staticmethod
7143
7144 @staticmethod
7166
7167 - def __init__(self, uri=DEFAULT_URI,
7168 pool_size=0, folder=None,
7169 db_codec='UTF-8', check_reserved=None,
7170 migrate=True, fake_migrate=False,
7171 migrate_enabled=True, fake_migrate_all=False,
7172 decode_credentials=False, driver_args=None,
7173 adapter_args=None, attempts=5, auto_import=False,
7174 bigint_id=False,debug=False,lazy_tables=False,
7175 db_uid=None, do_connect=True, after_connection=None):
7176 """
7177 Creates a new Database Abstraction Layer instance.
7178
7179 Keyword arguments:
7180
7181 :uri: string that contains information for connecting to a database.
7182 (default: 'sqlite://dummy.db')
7183 :pool_size: How many open connections to make to the database object.
7184 :folder: where .table files will be created.
7185 automatically set within web2py
7186 use an explicit path when using DAL outside web2py
7187 :db_codec: string encoding of the database (default: 'UTF-8')
7188 :check_reserved: list of adapters to check tablenames and column names
7189 against sql/nosql reserved keywords. (Default None)
7190
7191 * 'common' List of sql keywords that are common to all database types
7192 such as "SELECT, INSERT". (recommended)
7193 * 'all' Checks against all known SQL keywords. (not recommended)
7194 <adaptername> Checks against the specific adapters list of keywords
7195 (recommended)
7196 * '<adaptername>_nonreserved' Checks against the specific adapters
7197 list of nonreserved keywords. (if available)
7198 :migrate (defaults to True) sets default migrate behavior for all tables
7199 :fake_migrate (defaults to False) sets default fake_migrate behavior for all tables
7200 :migrate_enabled (defaults to True). If set to False disables ALL migrations
7201 :fake_migrate_all (defaults to False). If sets to True fake migrates ALL tables
7202 :attempts (defaults to 5). Number of times to attempt connecting
7203 :auto_import (defaults to False). If set, import automatically table definitions from the
7204 databases folder
7205 :bigint_id (defaults to False): If set, turn on bigint instead of int for id fields
7206 :lazy_tables (defaults to False): delay table definition until table access
7207 :after_connection (defaults to None): a callable that will be execute after the connection
7208 """
7209
7210 items = None
7211 if isinstance(uri, dict):
7212 if "items" in uri:
7213 items = uri.pop("items")
7214 try:
7215 newuri = uri.pop("uri")
7216 except KeyError:
7217 newuri = DEFAULT_URI
7218 locals().update(uri)
7219 uri = newuri
7220
7221 if uri == '<zombie>' and db_uid is not None: return
7222 if not decode_credentials:
7223 credential_decoder = lambda cred: cred
7224 else:
7225 credential_decoder = lambda cred: urllib.unquote(cred)
7226 self._folder = folder
7227 if folder:
7228 self.set_folder(folder)
7229 self._uri = uri
7230 self._pool_size = pool_size
7231 self._db_codec = db_codec
7232 self._lastsql = ''
7233 self._timings = []
7234 self._pending_references = {}
7235 self._request_tenant = 'request_tenant'
7236 self._common_fields = []
7237 self._referee_name = '%(table)s'
7238 self._bigint_id = bigint_id
7239 self._debug = debug
7240 self._migrated = []
7241 self._LAZY_TABLES = {}
7242 self._lazy_tables = lazy_tables
7243 self._tables = SQLCallableList()
7244 self._driver_args = driver_args
7245 self._adapter_args = adapter_args
7246 self._check_reserved = check_reserved
7247 self._decode_credentials = decode_credentials
7248 self._attempts = attempts
7249 self._do_connect = do_connect
7250
7251 if not str(attempts).isdigit() or attempts < 0:
7252 attempts = 5
7253 if uri:
7254 uris = isinstance(uri,(list,tuple)) and uri or [uri]
7255 error = ''
7256 connected = False
7257 for k in range(attempts):
7258 for uri in uris:
7259 try:
7260 if is_jdbc and not uri.startswith('jdbc:'):
7261 uri = 'jdbc:'+uri
7262 self._dbname = REGEX_DBNAME.match(uri).group()
7263 if not self._dbname in ADAPTERS:
7264 raise SyntaxError("Error in URI '%s' or database not supported" % self._dbname)
7265
7266
7267 kwargs = dict(db=self,uri=uri,
7268 pool_size=pool_size,
7269 folder=folder,
7270 db_codec=db_codec,
7271 credential_decoder=credential_decoder,
7272 driver_args=driver_args or {},
7273 adapter_args=adapter_args or {},
7274 do_connect=do_connect,
7275 after_connection=after_connection)
7276 self._adapter = ADAPTERS[self._dbname](**kwargs)
7277 types = ADAPTERS[self._dbname].types
7278
7279 self._adapter.types = copy.copy(types)
7280 if bigint_id:
7281 if 'big-id' in types and 'reference' in types:
7282 self._adapter.types['id'] = types['big-id']
7283 self._adapter.types['reference'] = types['big-reference']
7284 connected = True
7285 break
7286 except SyntaxError:
7287 raise
7288 except Exception:
7289 tb = traceback.format_exc()
7290 sys.stderr.write('DEBUG: connect attempt %i, connection error:\n%s' % (k, tb))
7291 if connected:
7292 break
7293 else:
7294 time.sleep(1)
7295 if not connected:
7296 raise RuntimeError("Failure to connect, tried %d times:\n%s" % (attempts, tb))
7297 else:
7298 self._adapter = BaseAdapter(db=self,pool_size=0,
7299 uri='None',folder=folder,
7300 db_codec=db_codec, after_connection=after_connection)
7301 migrate = fake_migrate = False
7302 adapter = self._adapter
7303 self._uri_hash = hashlib_md5(adapter.uri).hexdigest()
7304 self.check_reserved = check_reserved
7305 if self.check_reserved:
7306 from reserved_sql_keywords import ADAPTERS as RSK
7307 self.RSK = RSK
7308 self._migrate = migrate
7309 self._fake_migrate = fake_migrate
7310 self._migrate_enabled = migrate_enabled
7311 self._fake_migrate_all = fake_migrate_all
7312 if auto_import or items:
7313 self.import_table_definitions(adapter.folder,
7314 items=items)
7315
7316 @property
7319
7322 pattern = pjoin(path,self._uri_hash+'_*.table')
7323 if items:
7324 for tablename, table in items.iteritems():
7325
7326 fields = []
7327
7328 [table.pop(name) for name in ("name", "fields") if
7329 name in table]
7330 if "items" in table:
7331 for fieldname, field in table.pop("items").iteritems():
7332
7333 [field.pop(key) for key in ("requires", "name",
7334 "compute", "colname") if key in field]
7335 fields.append(Field(str(fieldname), **field))
7336 self.define_table(str(tablename), *fields, **table)
7337 else:
7338 for filename in glob.glob(pattern):
7339 tfile = self._adapter.file_open(filename, 'r')
7340 try:
7341 sql_fields = pickle.load(tfile)
7342 name = filename[len(pattern)-7:-6]
7343 mf = [(value['sortable'],
7344 Field(key,
7345 type=value['type'],
7346 length=value.get('length',None),
7347 notnull=value.get('notnull',False),
7348 unique=value.get('unique',False))) \
7349 for key, value in sql_fields.iteritems()]
7350 mf.sort(lambda a,b: cmp(a[0],b[0]))
7351 self.define_table(name,*[item[1] for item in mf],
7352 **dict(migrate=migrate,
7353 fake_migrate=fake_migrate))
7354 finally:
7355 self._adapter.file_close(tfile)
7356
7358 """
7359 Validates ``name`` against SQL keywords
7360 Uses self.check_reserve which is a list of
7361 operators to use.
7362 self.check_reserved
7363 ['common', 'postgres', 'mysql']
7364 self.check_reserved
7365 ['all']
7366 """
7367 for backend in self.check_reserved:
7368 if name.upper() in self.RSK[backend]:
7369 raise SyntaxError(
7370 'invalid table/column name "%s" is a "%s" reserved SQL/NOSQL keyword' % (name, backend.upper()))
7371
7372 - def parse_as_rest(self,patterns,args,vars,queries=None,nested_select=True):
7373 """
7374 EXAMPLE:
7375
7376 db.define_table('person',Field('name'),Field('info'))
7377 db.define_table('pet',Field('ownedby',db.person),Field('name'),Field('info'))
7378
7379 @request.restful()
7380 def index():
7381 def GET(*args,**vars):
7382 patterns = [
7383 "/friends[person]",
7384 "/{person.name}/:field",
7385 "/{person.name}/pets[pet.ownedby]",
7386 "/{person.name}/pets[pet.ownedby]/{pet.name}",
7387 "/{person.name}/pets[pet.ownedby]/{pet.name}/:field",
7388 ("/dogs[pet]", db.pet.info=='dog'),
7389 ("/dogs[pet]/{pet.name.startswith}", db.pet.info=='dog'),
7390 ]
7391 parser = db.parse_as_rest(patterns,args,vars)
7392 if parser.status == 200:
7393 return dict(content=parser.response)
7394 else:
7395 raise HTTP(parser.status,parser.error)
7396
7397 def POST(table_name,**vars):
7398 if table_name == 'person':
7399 return db.person.validate_and_insert(**vars)
7400 elif table_name == 'pet':
7401 return db.pet.validate_and_insert(**vars)
7402 else:
7403 raise HTTP(400)
7404 return locals()
7405 """
7406
7407 db = self
7408 re1 = REGEX_SEARCH_PATTERN
7409 re2 = REGEX_SQUARE_BRACKETS
7410
7411 def auto_table(table,base='',depth=0):
7412 patterns = []
7413 for field in db[table].fields:
7414 if base:
7415 tag = '%s/%s' % (base,field.replace('_','-'))
7416 else:
7417 tag = '/%s/%s' % (table.replace('_','-'),field.replace('_','-'))
7418 f = db[table][field]
7419 if not f.readable: continue
7420 if f.type=='id' or 'slug' in field or f.type.startswith('reference'):
7421 tag += '/{%s.%s}' % (table,field)
7422 patterns.append(tag)
7423 patterns.append(tag+'/:field')
7424 elif f.type.startswith('boolean'):
7425 tag += '/{%s.%s}' % (table,field)
7426 patterns.append(tag)
7427 patterns.append(tag+'/:field')
7428 elif f.type in ('float','double','integer','bigint'):
7429 tag += '/{%s.%s.ge}/{%s.%s.lt}' % (table,field,table,field)
7430 patterns.append(tag)
7431 patterns.append(tag+'/:field')
7432 elif f.type.startswith('list:'):
7433 tag += '/{%s.%s.contains}' % (table,field)
7434 patterns.append(tag)
7435 patterns.append(tag+'/:field')
7436 elif f.type in ('date','datetime'):
7437 tag+= '/{%s.%s.year}' % (table,field)
7438 patterns.append(tag)
7439 patterns.append(tag+'/:field')
7440 tag+='/{%s.%s.month}' % (table,field)
7441 patterns.append(tag)
7442 patterns.append(tag+'/:field')
7443 tag+='/{%s.%s.day}' % (table,field)
7444 patterns.append(tag)
7445 patterns.append(tag+'/:field')
7446 if f.type in ('datetime','time'):
7447 tag+= '/{%s.%s.hour}' % (table,field)
7448 patterns.append(tag)
7449 patterns.append(tag+'/:field')
7450 tag+='/{%s.%s.minute}' % (table,field)
7451 patterns.append(tag)
7452 patterns.append(tag+'/:field')
7453 tag+='/{%s.%s.second}' % (table,field)
7454 patterns.append(tag)
7455 patterns.append(tag+'/:field')
7456 if depth>0:
7457 for f in db[table]._referenced_by:
7458 tag+='/%s[%s.%s]' % (table,f.tablename,f.name)
7459 patterns.append(tag)
7460 patterns += auto_table(table,base=tag,depth=depth-1)
7461 return patterns
7462
7463 if patterns == 'auto':
7464 patterns=[]
7465 for table in db.tables:
7466 if not table.startswith('auth_'):
7467 patterns.append('/%s[%s]' % (table,table))
7468 patterns += auto_table(table,base='',depth=1)
7469 else:
7470 i = 0
7471 while i<len(patterns):
7472 pattern = patterns[i]
7473 if not isinstance(pattern,str):
7474 pattern = pattern[0]
7475 tokens = pattern.split('/')
7476 if tokens[-1].startswith(':auto') and re2.match(tokens[-1]):
7477 new_patterns = auto_table(tokens[-1][tokens[-1].find('[')+1:-1],
7478 '/'.join(tokens[:-1]))
7479 patterns = patterns[:i]+new_patterns+patterns[i+1:]
7480 i += len(new_patterns)
7481 else:
7482 i += 1
7483 if '/'.join(args) == 'patterns':
7484 return Row({'status':200,'pattern':'list',
7485 'error':None,'response':patterns})
7486 for pattern in patterns:
7487 basequery, exposedfields = None, []
7488 if isinstance(pattern,tuple):
7489 if len(pattern)==2:
7490 pattern, basequery = pattern
7491 elif len(pattern)>2:
7492 pattern, basequery, exposedfields = pattern[0:3]
7493 otable=table=None
7494 if not isinstance(queries,dict):
7495 dbset=db(queries)
7496 if basequery is not None:
7497 dbset = dbset(basequery)
7498 i=0
7499 tags = pattern[1:].split('/')
7500 if len(tags)!=len(args):
7501 continue
7502 for tag in tags:
7503 if re1.match(tag):
7504
7505 tokens = tag[1:-1].split('.')
7506 table, field = tokens[0], tokens[1]
7507 if not otable or table == otable:
7508 if len(tokens)==2 or tokens[2]=='eq':
7509 query = db[table][field]==args[i]
7510 elif tokens[2]=='ne':
7511 query = db[table][field]!=args[i]
7512 elif tokens[2]=='lt':
7513 query = db[table][field]<args[i]
7514 elif tokens[2]=='gt':
7515 query = db[table][field]>args[i]
7516 elif tokens[2]=='ge':
7517 query = db[table][field]>=args[i]
7518 elif tokens[2]=='le':
7519 query = db[table][field]<=args[i]
7520 elif tokens[2]=='year':
7521 query = db[table][field].year()==args[i]
7522 elif tokens[2]=='month':
7523 query = db[table][field].month()==args[i]
7524 elif tokens[2]=='day':
7525 query = db[table][field].day()==args[i]
7526 elif tokens[2]=='hour':
7527 query = db[table][field].hour()==args[i]
7528 elif tokens[2]=='minute':
7529 query = db[table][field].minutes()==args[i]
7530 elif tokens[2]=='second':
7531 query = db[table][field].seconds()==args[i]
7532 elif tokens[2]=='startswith':
7533 query = db[table][field].startswith(args[i])
7534 elif tokens[2]=='contains':
7535 query = db[table][field].contains(args[i])
7536 else:
7537 raise RuntimeError("invalid pattern: %s" % pattern)
7538 if len(tokens)==4 and tokens[3]=='not':
7539 query = ~query
7540 elif len(tokens)>=4:
7541 raise RuntimeError("invalid pattern: %s" % pattern)
7542 if not otable and isinstance(queries,dict):
7543 dbset = db(queries[table])
7544 if basequery is not None:
7545 dbset = dbset(basequery)
7546 dbset=dbset(query)
7547 else:
7548 raise RuntimeError("missing relation in pattern: %s" % pattern)
7549 elif re2.match(tag) and args[i]==tag[:tag.find('[')]:
7550 ref = tag[tag.find('[')+1:-1]
7551 if '.' in ref and otable:
7552 table,field = ref.split('.')
7553 selfld = '_id'
7554 if db[table][field].type.startswith('reference '):
7555 refs = [ x.name for x in db[otable] if x.type == db[table][field].type ]
7556 else:
7557 refs = [ x.name for x in db[table]._referenced_by if x.tablename==otable ]
7558 if refs:
7559 selfld = refs[0]
7560 if nested_select:
7561 try:
7562 dbset=db(db[table][field].belongs(dbset._select(db[otable][selfld])))
7563 except ValueError:
7564 return Row({'status':400,'pattern':pattern,
7565 'error':'invalid path','response':None})
7566 else:
7567 items = [item.id for item in dbset.select(db[otable][selfld])]
7568 dbset=db(db[table][field].belongs(items))
7569 else:
7570 table = ref
7571 if not otable and isinstance(queries,dict):
7572 dbset = db(queries[table])
7573 dbset=dbset(db[table])
7574 elif tag==':field' and table:
7575
7576 field = args[i]
7577 if not field in db[table]: break
7578
7579 if not db[table][field].readable:
7580 return Row({'status':418,'pattern':pattern,
7581 'error':'I\'m a teapot','response':None})
7582 try:
7583 distinct = vars.get('distinct', False) == 'True'
7584 offset = int(vars.get('offset',None) or 0)
7585 limits = (offset,int(vars.get('limit',None) or 1000)+offset)
7586 except ValueError:
7587 return Row({'status':400,'error':'invalid limits','response':None})
7588 items = dbset.select(db[table][field], distinct=distinct, limitby=limits)
7589 if items:
7590 return Row({'status':200,'response':items,
7591 'pattern':pattern})
7592 else:
7593 return Row({'status':404,'pattern':pattern,
7594 'error':'no record found','response':None})
7595 elif tag != args[i]:
7596 break
7597 otable = table
7598 i += 1
7599 if i==len(tags) and table:
7600 ofields = vars.get('order',db[table]._id.name).split('|')
7601 try:
7602 orderby = [db[table][f] if not f.startswith('~') else ~db[table][f[1:]] for f in ofields]
7603 except (KeyError, AttributeError):
7604 return Row({'status':400,'error':'invalid orderby','response':None})
7605 if exposedfields:
7606 fields = [field for field in db[table] if str(field).split('.')[-1] in exposedfields and field.readable]
7607 else:
7608 fields = [field for field in db[table] if field.readable]
7609 count = dbset.count()
7610 try:
7611 offset = int(vars.get('offset',None) or 0)
7612 limits = (offset,int(vars.get('limit',None) or 1000)+offset)
7613 except ValueError:
7614 return Row({'status':400,'error':'invalid limits','response':None})
7615 if count > limits[1]-limits[0]:
7616 return Row({'status':400,'error':'too many records','response':None})
7617 try:
7618 response = dbset.select(limitby=limits,orderby=orderby,*fields)
7619 except ValueError:
7620 return Row({'status':400,'pattern':pattern,
7621 'error':'invalid path','response':None})
7622 return Row({'status':200,'response':response,
7623 'pattern':pattern,'count':count})
7624 return Row({'status':400,'error':'no matching pattern','response':None})
7625
7626 - def define_table(
7627 self,
7628 tablename,
7629 *fields,
7630 **args
7631 ):
7632 if not isinstance(tablename,str):
7633 raise SyntaxError("missing table name")
7634 elif hasattr(self,tablename) or tablename in self.tables:
7635 if not args.get('redefine',False):
7636 raise SyntaxError('table already defined: %s' % tablename)
7637 elif tablename.startswith('_') or hasattr(self,tablename) or \
7638 REGEX_PYTHON_KEYWORDS.match(tablename):
7639 raise SyntaxError('invalid table name: %s' % tablename)
7640 elif self.check_reserved:
7641 self.check_reserved_keyword(tablename)
7642 else:
7643 invalid_args = set(args)-TABLE_ARGS
7644 if invalid_args:
7645 raise SyntaxError('invalid table "%s" attributes: %s' \
7646 % (tablename,invalid_args))
7647 if self._lazy_tables and not tablename in self._LAZY_TABLES:
7648 self._LAZY_TABLES[tablename] = (tablename,fields,args)
7649 table = None
7650 else:
7651 table = self.lazy_define_table(tablename,*fields,**args)
7652 if not tablename in self.tables:
7653 self.tables.append(tablename)
7654 return table
7655
7656 - def lazy_define_table(
7657 self,
7658 tablename,
7659 *fields,
7660 **args
7661 ):
7662 args_get = args.get
7663 common_fields = self._common_fields
7664 if common_fields:
7665 fields = list(fields) + list(common_fields)
7666
7667 table_class = args_get('table_class',Table)
7668 table = table_class(self, tablename, *fields, **args)
7669 table._actual = True
7670 self[tablename] = table
7671
7672 table._create_references()
7673 for field in table:
7674 if field.requires == DEFAULT:
7675 field.requires = sqlhtml_validators(field)
7676
7677 migrate = self._migrate_enabled and args_get('migrate',self._migrate)
7678 if migrate and not self._uri in (None,'None') \
7679 or self._adapter.dbengine=='google:datastore':
7680 fake_migrate = self._fake_migrate_all or \
7681 args_get('fake_migrate',self._fake_migrate)
7682 polymodel = args_get('polymodel',None)
7683 try:
7684 GLOBAL_LOCKER.acquire()
7685 self._lastsql = self._adapter.create_table(
7686 table,migrate=migrate,
7687 fake_migrate=fake_migrate,
7688 polymodel=polymodel)
7689 finally:
7690 GLOBAL_LOCKER.release()
7691 else:
7692 table._dbt = None
7693 on_define = args_get('on_define',None)
7694 if on_define: on_define(table)
7695 return table
7696
7697 - def as_dict(self, flat=False, sanitize=True, field_options=True):
7698 dbname = db_uid = uri = None
7699 if not sanitize:
7700 uri, dbname, db_uid = (self._uri, self._dbname, self._db_uid)
7701 db_as_dict = dict(items={}, tables=[], uri=uri, dbname=dbname,
7702 db_uid=db_uid,
7703 **dict([(k, getattr(self, "_" + k)) for
7704 k in 'pool_size','folder','db_codec',
7705 'check_reserved','migrate','fake_migrate',
7706 'migrate_enabled','fake_migrate_all',
7707 'decode_credentials','driver_args',
7708 'adapter_args', 'attempts',
7709 'bigint_id','debug','lazy_tables',
7710 'do_connect']))
7711
7712 for table in self:
7713 tablename = str(table)
7714 db_as_dict["tables"].append(tablename)
7715 db_as_dict["items"][tablename] = table.as_dict(flat=flat,
7716 sanitize=sanitize,
7717 field_options=field_options)
7718 return db_as_dict
7719
7720 - def as_xml(self, sanitize=True, field_options=True):
7726
7727 - def as_json(self, sanitize=True, field_options=True):
7733
7734 - def as_yaml(self, sanitize=True, field_options=True):
7740
7742 try:
7743 return tablename in self.tables
7744 except AttributeError:
7745
7746 return False
7747
7748 has_key = __contains__
7749
7750 - def get(self,key,default=None):
7751 return self.__dict__.get(key,default)
7752
7754 for tablename in self.tables:
7755 yield self[tablename]
7756
7759
7761 if ogetattr(self,'_lazy_tables') and \
7762 key in ogetattr(self,'_LAZY_TABLES'):
7763 tablename, fields, args = self._LAZY_TABLES.pop(key)
7764 return self.lazy_define_table(tablename,*fields,**args)
7765 return ogetattr(self, key)
7766
7768 osetattr(self, str(key), value)
7769
7771 if key[:1]!='_' and key in self:
7772 raise SyntaxError(
7773 'Object %s exists and cannot be redefined' % key)
7774 osetattr(self,key,value)
7775
7776 __delitem__ = object.__delattr__
7777
7779 if hasattr(self,'_uri'):
7780 return '<DAL uri="%s">' % hide_password(str(self._uri))
7781 else:
7782 return '<DAL db_uid="%s">' % self._db_uid
7783
7786
7787 - def __call__(self, query=None, ignore_common_filters=None):
7788 if isinstance(query,Table):
7789 query = self._adapter.id_query(query)
7790 elif isinstance(query,Field):
7791 query = query!=None
7792 elif isinstance(query, dict):
7793 icf = query.get("ignore_common_filters")
7794 if icf: ignore_common_filters = icf
7795 return Set(self, query, ignore_common_filters=ignore_common_filters)
7796
7799
7802
7804 self._adapter.close()
7805 if self._db_uid in THREAD_LOCAL.db_instances:
7806 db_group = THREAD_LOCAL.db_instances[self._db_uid]
7807 db_group.remove(self)
7808 if not db_group:
7809 del THREAD_LOCAL.db_instances[self._db_uid]
7810
7811 - def executesql(self, query, placeholders=None, as_dict=False,
7812 fields=None, colnames=None):
7813 """
7814 placeholders is optional and will always be None.
7815 If using raw SQL with placeholders, placeholders may be
7816 a sequence of values to be substituted in
7817 or, (if supported by the DB driver), a dictionary with keys
7818 matching named placeholders in your SQL.
7819
7820 Added 2009-12-05 "as_dict" optional argument. Will always be
7821 None when using DAL. If using raw SQL can be set to True
7822 and the results cursor returned by the DB driver will be
7823 converted to a sequence of dictionaries keyed with the db
7824 field names. Tested with SQLite but should work with any database
7825 since the cursor.description used to get field names is part of the
7826 Python dbi 2.0 specs. Results returned with as_dict=True are
7827 the same as those returned when applying .to_list() to a DAL query.
7828
7829 [{field1: value1, field2: value2}, {field1: value1b, field2: value2b}]
7830
7831 Added 2012-08-24 "fields" and "colnames" optional arguments. If either
7832 is provided, the results cursor returned by the DB driver will be
7833 converted to a DAL Rows object using the db._adapter.parse() method.
7834
7835 The "fields" argument is a list of DAL Field objects that match the
7836 fields returned from the DB. The Field objects should be part of one or
7837 more Table objects defined on the DAL object. The "fields" list can
7838 include one or more DAL Table objects in addition to or instead of
7839 including Field objects, or it can be just a single table (not in a
7840 list). In that case, the Field objects will be extracted from the
7841 table(s).
7842
7843 Instead of specifying the "fields" argument, the "colnames" argument
7844 can be specified as a list of field names in tablename.fieldname format.
7845 Again, these should represent tables and fields defined on the DAL
7846 object.
7847
7848 It is also possible to specify both "fields" and the associated
7849 "colnames". In that case, "fields" can also include DAL Expression
7850 objects in addition to Field objects. For Field objects in "fields",
7851 the associated "colnames" must still be in tablename.fieldname format.
7852 For Expression objects in "fields", the associated "colnames" can
7853 be any arbitrary labels.
7854
7855 Note, the DAL Table objects referred to by "fields" or "colnames" can
7856 be dummy tables and do not have to represent any real tables in the
7857 database. Also, note that the "fields" and "colnames" must be in the
7858 same order as the fields in the results cursor returned from the DB.
7859 """
7860 adapter = self._adapter
7861 if placeholders:
7862 adapter.execute(query, placeholders)
7863 else:
7864 adapter.execute(query)
7865 if as_dict:
7866 if not hasattr(adapter.cursor,'description'):
7867 raise RuntimeError("database does not support executesql(...,as_dict=True)")
7868
7869
7870
7871 columns = adapter.cursor.description
7872
7873 fields = [f[0] for f in columns]
7874
7875 data = adapter._fetchall()
7876
7877
7878 return [dict(zip(fields,row)) for row in data]
7879 try:
7880 data = adapter._fetchall()
7881 except:
7882 return None
7883 if fields or colnames:
7884 fields = [] if fields is None else fields
7885 if not isinstance(fields, list):
7886 fields = [fields]
7887 extracted_fields = []
7888 for field in fields:
7889 if isinstance(field, Table):
7890 extracted_fields.extend([f for f in field])
7891 else:
7892 extracted_fields.append(field)
7893 if not colnames:
7894 colnames = ['%s.%s' % (f.tablename, f.name)
7895 for f in extracted_fields]
7896 data = adapter.parse(
7897 data, fields=extracted_fields, colnames=colnames)
7898 return data
7899
7901 for table in self:
7902 table._referenced_by = [field for field in table._referenced_by
7903 if not field.table==thistable]
7904
7906 step = int(kwargs.get('max_fetch_rows,',500))
7907 write_colnames = kwargs['write_colnames'] = \
7908 kwargs.get("write_colnames", True)
7909 for table in self.tables:
7910 ofile.write('TABLE %s\r\n' % table)
7911 query = self._adapter.id_query(self[table])
7912 nrows = self(query).count()
7913 kwargs['write_colnames'] = write_colnames
7914 for k in range(0,nrows,step):
7915 self(query).select(limitby=(k,k+step)).export_to_csv_file(
7916 ofile, *args, **kwargs)
7917 kwargs['write_colnames'] = False
7918 ofile.write('\r\n\r\n')
7919 ofile.write('END')
7920
7921 - def import_from_csv_file(self, ifile, id_map=None, null='<NULL>',
7922 unique='uuid', *args, **kwargs):
7923
7924 id_offset = {}
7925 for line in ifile:
7926 line = line.strip()
7927 if not line:
7928 continue
7929 elif line == 'END':
7930 return
7931 elif not line.startswith('TABLE ') or not line[6:] in self.tables:
7932 raise SyntaxError('invalid file format')
7933 else:
7934 tablename = line[6:]
7935 self[tablename].import_from_csv_file(
7936 ifile, id_map, null, unique, id_offset, *args, **kwargs)
7937
7939 return DAL('<zombie>',db_uid=db_uid)
7940
7943
7944 copyreg.pickle(DAL, DAL_pickler, DAL_unpickler)
7947 """
7948 Helper class providing a comma-separated string having all the field names
7949 (prefixed by table name and '.')
7950
7951 normally only called from within gluon.sql
7952 """
7953
7956
7958 return ', '.join([str(field) for field in self._table])
7959
7962
7964 if not self._record:
7965 self._record = self._table[int(self)]
7966 if not self._record:
7967 raise RuntimeError(
7968 "Using a recursive select but encountered a broken reference: %s %d"%(self._table, int(self)))
7969
7971 if key == 'id':
7972 return int(self)
7973 self.__allocate()
7974 return self._record.get(key, None)
7975
7976 - def get(self, key, default=None):
7978
7985
7987 if key == 'id':
7988 return int(self)
7989 self.__allocate()
7990 return self._record.get(key, None)
7991
7993 self.__allocate()
7994 self._record[key] = value
7995
7998 return marshal.loads(data)
7999
8001 try:
8002 marshal_dump = marshal.dumps(int(data))
8003 except AttributeError:
8004 marshal_dump = 'i%s' % struct.pack('<i', int(data))
8005 return (Reference_unpickler, (marshal_dump,))
8006
8007 copyreg.pickle(Reference, Reference_pickler, Reference_unpickler)
8008
8009
8010 -class Table(object):
8011
8012 """
8013 an instance of this class represents a database table
8014
8015 Example::
8016
8017 db = DAL(...)
8018 db.define_table('users', Field('name'))
8019 db.users.insert(name='me') # print db.users._insert(...) to see SQL
8020 db.users.drop()
8021 """
8022
8023 - def __init__(
8024 self,
8025 db,
8026 tablename,
8027 *fields,
8028 **args
8029 ):
8030 """
8031 Initializes the table and performs checking on the provided fields.
8032
8033 Each table will have automatically an 'id'.
8034
8035 If a field is of type Table, the fields (excluding 'id') from that table
8036 will be used instead.
8037
8038 :raises SyntaxError: when a supplied field is of incorrect type.
8039 """
8040 self._actual = False
8041 self._tablename = tablename
8042 self._sequence_name = args.get('sequence_name',None) or \
8043 db and db._adapter.sequence_name(tablename)
8044 self._trigger_name = args.get('trigger_name',None) or \
8045 db and db._adapter.trigger_name(tablename)
8046 self._common_filter = args.get('common_filter', None)
8047 self._format = args.get('format',None)
8048 self._singular = args.get(
8049 'singular',tablename.replace('_',' ').capitalize())
8050 self._plural = args.get(
8051 'plural',pluralize(self._singular.lower()).capitalize())
8052
8053 if 'primarykey' in args and args['primarykey']:
8054 self._primarykey = args.get('primarykey', None)
8055
8056 self._before_insert = []
8057 self._before_update = [Set.delete_uploaded_files]
8058 self._before_delete = [Set.delete_uploaded_files]
8059 self._after_insert = []
8060 self._after_update = []
8061 self._after_delete = []
8062
8063 fieldnames,newfields=set(),[]
8064 if hasattr(self,'_primarykey'):
8065 if not isinstance(self._primarykey,list):
8066 raise SyntaxError(
8067 "primarykey must be a list of fields from table '%s'" \
8068 % tablename)
8069 if len(self._primarykey)==1:
8070 self._id = [f for f in fields if isinstance(f,Field) \
8071 and f.name==self._primarykey[0]][0]
8072 elif not [f for f in fields if isinstance(f,Field) and f.type=='id']:
8073 field = Field('id', 'id')
8074 newfields.append(field)
8075 fieldnames.add('id')
8076 self._id = field
8077 virtual_fields = []
8078 for field in fields:
8079 if isinstance(field, (FieldMethod, FieldVirtual)):
8080 virtual_fields.append(field)
8081 elif isinstance(field, Field) and not field.name in fieldnames:
8082 if field.db is not None:
8083 field = copy.copy(field)
8084 newfields.append(field)
8085 fieldnames.add(field.name)
8086 if field.type=='id':
8087 self._id = field
8088 elif isinstance(field, Table):
8089 table = field
8090 for field in table:
8091 if not field.name in fieldnames and not field.type=='id':
8092 t2 = not table._actual and self._tablename
8093 field = field.clone(point_self_references_to=t2)
8094 newfields.append(field)
8095 fieldnames.add(field.name)
8096 elif not isinstance(field, (Field, Table)):
8097 raise SyntaxError(
8098 'define_table argument is not a Field or Table: %s' % field)
8099 fields = newfields
8100 self._db = db
8101 tablename = tablename
8102 self._fields = SQLCallableList()
8103 self.virtualfields = []
8104 fields = list(fields)
8105
8106 if db and db._adapter.uploads_in_blob==True:
8107 uploadfields = [f.name for f in fields if f.type=='blob']
8108 for field in fields:
8109 fn = field.uploadfield
8110 if isinstance(field, Field) and field.type == 'upload'\
8111 and fn is True:
8112 fn = field.uploadfield = '%s_blob' % field.name
8113 if isinstance(fn,str) and not fn in uploadfields:
8114 fields.append(Field(fn,'blob',default='',
8115 writable=False,readable=False))
8116
8117 lower_fieldnames = set()
8118 reserved = dir(Table) + ['fields']
8119 for field in fields:
8120 field_name = field.name
8121 if db and db.check_reserved:
8122 db.check_reserved_keyword(field_name)
8123 elif field_name in reserved:
8124 raise SyntaxError("field name %s not allowed" % field_name)
8125
8126 if field_name.lower() in lower_fieldnames:
8127 raise SyntaxError("duplicate field %s in table %s" \
8128 % (field_name, tablename))
8129 else:
8130 lower_fieldnames.add(field_name.lower())
8131
8132 self.fields.append(field_name)
8133 self[field_name] = field
8134 if field.type == 'id':
8135 self['id'] = field
8136 field.tablename = field._tablename = tablename
8137 field.table = field._table = self
8138 field.db = field._db = db
8139 if db and not field.type in ('text', 'blob', 'json') and \
8140 db._adapter.maxcharlength < field.length:
8141 field.length = db._adapter.maxcharlength
8142 self.ALL = SQLALL(self)
8143
8144 if hasattr(self,'_primarykey'):
8145 for k in self._primarykey:
8146 if k not in self.fields:
8147 raise SyntaxError(
8148 "primarykey must be a list of fields from table '%s " % tablename)
8149 else:
8150 self[k].notnull = True
8151 for field in virtual_fields:
8152 self[field.name] = field
8153
8154 @property
8157
8158 - def update(self,*args,**kwargs):
8159 raise RuntimeError("Syntax Not Supported")
8160
8161 - def _enable_record_versioning(self,
8162 archive_db=None,
8163 archive_name = '%(tablename)s_archive',
8164 current_record = 'current_record',
8165 is_active = 'is_active'):
8166 archive_db = archive_db or self._db
8167 archive_name = archive_name % dict(tablename=self._tablename)
8168 if archive_name in archive_db.tables():
8169 return
8170 fieldnames = self.fields()
8171 field_type = self if archive_db is self._db else 'bigint'
8172 archive_db.define_table(
8173 archive_name,
8174 Field(current_record,field_type),
8175 *[field.clone(unique=False) for field in self])
8176 self._before_update.append(
8177 lambda qset,fs,db=archive_db,an=archive_name,cn=current_record:
8178 archive_record(qset,fs,db[an],cn))
8179 if is_active and is_active in fieldnames:
8180 self._before_delete.append(
8181 lambda qset: qset.update(is_active=False))
8182 newquery = lambda query, t=self: t.is_active == True
8183 query = self._common_filter
8184 if query:
8185 newquery = query & newquery
8186 self._common_filter = newquery
8187
8195
8197 db = self._db
8198 pr = db._pending_references
8199 self._referenced_by = []
8200 for field in self:
8201 fieldname = field.name
8202 field_type = field.type
8203 if isinstance(field_type,str) and field_type[:10] == 'reference ':
8204 ref = field_type[10:].strip()
8205 if not ref.split():
8206 raise SyntaxError('Table: reference to nothing: %s' %ref)
8207 refs = ref.split('.')
8208 rtablename = refs[0]
8209 if not rtablename in db:
8210 pr[rtablename] = pr.get(rtablename,[]) + [field]
8211 continue
8212 rtable = db[rtablename]
8213 if len(refs)==2:
8214 rfieldname = refs[1]
8215 if not hasattr(rtable,'_primarykey'):
8216 raise SyntaxError(
8217 'keyed tables can only reference other keyed tables (for now)')
8218 if rfieldname not in rtable.fields:
8219 raise SyntaxError(
8220 "invalid field '%s' for referenced table '%s' in table '%s'" \
8221 % (rfieldname, rtablename, self._tablename))
8222 rtable._referenced_by.append(field)
8223 for referee in pr.get(self._tablename,[]):
8224 self._referenced_by.append(referee)
8225
8227 return dict([(k, v) for (k, v) in record.iteritems() if k
8228 in self.fields and (self[k].type!='id' or id)])
8229
8231 """ for keyed table only """
8232 query = None
8233 for k,v in key.iteritems():
8234 if k in self._primarykey:
8235 if query:
8236 query = query & (self[k] == v)
8237 else:
8238 query = (self[k] == v)
8239 else:
8240 raise SyntaxError(
8241 'Field %s is not part of the primary key of %s' % \
8242 (k,self._tablename))
8243 return query
8244
8246 if not key:
8247 return None
8248 elif isinstance(key, dict):
8249 """ for keyed table """
8250 query = self._build_query(key)
8251 rows = self._db(query).select()
8252 if rows:
8253 return rows[0]
8254 return None
8255 elif str(key).isdigit() or 'google' in DRIVERS and isinstance(key, Key):
8256 return self._db(self._id == key).select(limitby=(0,1)).first()
8257 elif key:
8258 return ogetattr(self, str(key))
8259
8261 for_update = kwargs.get('_for_update',False)
8262 if '_for_update' in kwargs: del kwargs['_for_update']
8263
8264 orderby = kwargs.get('_orderby',None)
8265 if '_orderby' in kwargs: del kwargs['_orderby']
8266
8267 if not key is DEFAULT:
8268 if isinstance(key, Query):
8269 record = self._db(key).select(
8270 limitby=(0,1),for_update=for_update, orderby=orderby).first()
8271 elif not str(key).isdigit():
8272 record = None
8273 else:
8274 record = self._db(self._id == key).select(
8275 limitby=(0,1),for_update=for_update, orderby=orderby).first()
8276 if record:
8277 for k,v in kwargs.iteritems():
8278 if record[k]!=v: return None
8279 return record
8280 elif kwargs:
8281 query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.iteritems()])
8282 return self._db(query).select(limitby=(0,1),for_update=for_update, orderby=orderby).first()
8283 else:
8284 return None
8285
8287 if isinstance(key, dict) and isinstance(value, dict):
8288 """ option for keyed table """
8289 if set(key.keys()) == set(self._primarykey):
8290 value = self._filter_fields(value)
8291 kv = {}
8292 kv.update(value)
8293 kv.update(key)
8294 if not self.insert(**kv):
8295 query = self._build_query(key)
8296 self._db(query).update(**self._filter_fields(value))
8297 else:
8298 raise SyntaxError(
8299 'key must have all fields from primary key: %s'%\
8300 (self._primarykey))
8301 elif str(key).isdigit():
8302 if key == 0:
8303 self.insert(**self._filter_fields(value))
8304 elif self._db(self._id == key)\
8305 .update(**self._filter_fields(value)) is None:
8306 raise SyntaxError('No such record: %s' % key)
8307 else:
8308 if isinstance(key, dict):
8309 raise SyntaxError(
8310 'value must be a dictionary: %s' % value)
8311 osetattr(self, str(key), value)
8312
8313 __getattr__ = __getitem__
8314
8316 if key[:1]!='_' and key in self:
8317 raise SyntaxError('Object exists and cannot be redefined: %s' % key)
8318 osetattr(self,key,value)
8319
8321 if isinstance(key, dict):
8322 query = self._build_query(key)
8323 if not self._db(query).delete():
8324 raise SyntaxError('No such record: %s' % key)
8325 elif not str(key).isdigit() or \
8326 not self._db(self._id == key).delete():
8327 raise SyntaxError('No such record: %s' % key)
8328
8330 return hasattr(self,key)
8331
8332 has_key = __contains__
8333
8335 return self.__dict__.items()
8336
8338 for fieldname in self.fields:
8339 yield self[fieldname]
8340
8343
8344
8346 return '<Table %s (%s)>' % (self._tablename,','.join(self.fields()))
8347
8349 if hasattr(self,'_ot') and self._ot is not None:
8350 if 'Oracle' in str(type(self._db._adapter)):
8351 return '%s %s' % (self._ot, self._tablename)
8352 return '%s AS %s' % (self._ot, self._tablename)
8353 return self._tablename
8354
8355 - def _drop(self, mode = ''):
8356 return self._db._adapter._drop(self, mode)
8357
8358 - def drop(self, mode = ''):
8359 return self._db._adapter.drop(self,mode)
8360
8361 - def _listify(self,fields,update=False):
8362 new_fields = {}
8363
8364
8365 for name in fields:
8366 if not name in self.fields:
8367 if name != 'id':
8368 raise SyntaxError(
8369 'Field %s does not belong to the table' % name)
8370 else:
8371 field = self[name]
8372 value = fields[name]
8373 if field.filter_in:
8374 value = field.filter_in(value)
8375 new_fields[name] = (field,value)
8376
8377
8378 to_compute = []
8379 for ofield in self:
8380 name = ofield.name
8381 if not name in new_fields:
8382
8383 if ofield.compute:
8384 to_compute.append((name,ofield))
8385
8386 elif not update and not ofield.default is None:
8387 value = ofield.default
8388 fields[name] = value
8389 new_fields[name] = (ofield,value)
8390
8391 elif update and not ofield.update is None:
8392 value = ofield.update
8393 fields[name] = value
8394 new_fields[name] = (ofield,value)
8395
8396 elif not update and ofield.required:
8397 raise RuntimeError(
8398 'Table: missing required field: %s' % name)
8399
8400 if to_compute:
8401 row = Row(fields)
8402 for name,ofield in to_compute:
8403
8404 try:
8405 new_fields[name] = (ofield,ofield.compute(row))
8406 except (KeyError, AttributeError):
8407
8408 if ofield.required:
8409 raise SyntaxError('unable to comput field: %s' % name)
8410 return new_fields.values()
8411
8413 for field in self:
8414 if field.type=='upload' and field.name in fields:
8415 value = fields[field.name]
8416 if value and not isinstance(value,str):
8417 if hasattr(value,'file') and hasattr(value,'filename'):
8418 new_name = field.store(value.file,filename=value.filename)
8419 elif hasattr(value,'read') and hasattr(value,'name'):
8420 new_name = field.store(value,filename=value.name)
8421 else:
8422 raise RuntimeError("Unable to handle upload")
8423 fields[field.name] = new_name
8424
8426 "If there are no fields/values specified, return table defaults"
8427 if not fields:
8428 fields = {}
8429 for field in self:
8430 if field.type != "id":
8431 fields[field.name] = field.default
8432 return fields
8433
8437
8447
8463
8465 if _key is DEFAULT:
8466 record = self(**values)
8467 elif isinstance(_key,dict):
8468 record = self(**_key)
8469 else:
8470 record = self(_key)
8471 if record:
8472 record.update_record(**values)
8473 newid = None
8474 else:
8475 newid = self.insert(**values)
8476 return newid
8477
8479 """
8480 here items is a list of dictionaries
8481 """
8482 items = [self._listify(item) for item in items]
8483 if any(f(item) for item in items for f in self._before_insert):return 0
8484 ret = self._db._adapter.bulk_insert(self,items)
8485 ret and [[f(item,ret[k]) for k,item in enumerate(items)] for f in self._after_insert]
8486 return ret
8487
8489 return self._db._adapter._truncate(self, mode)
8490
8492 return self._db._adapter.truncate(self, mode)
8493
8494 - def import_from_csv_file(
8495 self,
8496 csvfile,
8497 id_map=None,
8498 null='<NULL>',
8499 unique='uuid',
8500 id_offset=None,
8501 *args, **kwargs
8502 ):
8503 """
8504 Import records from csv file.
8505 Column headers must have same names as table fields.
8506 Field 'id' is ignored.
8507 If column names read 'table.file' the 'table.' prefix is ignored.
8508 'unique' argument is a field which must be unique
8509 (typically a uuid field)
8510 'restore' argument is default False;
8511 if set True will remove old values in table first.
8512 'id_map' ff set to None will not map ids.
8513 The import will keep the id numbers in the restored table.
8514 This assumes that there is an field of type id that
8515 is integer and in incrementing order.
8516 Will keep the id numbers in restored table.
8517 """
8518
8519 delimiter = kwargs.get('delimiter', ',')
8520 quotechar = kwargs.get('quotechar', '"')
8521 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
8522 restore = kwargs.get('restore', False)
8523 if restore:
8524 self._db[self].truncate()
8525
8526 reader = csv.reader(csvfile, delimiter=delimiter,
8527 quotechar=quotechar, quoting=quoting)
8528 colnames = None
8529 if isinstance(id_map, dict):
8530 if not self._tablename in id_map:
8531 id_map[self._tablename] = {}
8532 id_map_self = id_map[self._tablename]
8533
8534 def fix(field, value, id_map, id_offset):
8535 list_reference_s='list:reference'
8536 if value == null:
8537 value = None
8538 elif field.type=='blob':
8539 value = base64.b64decode(value)
8540 elif field.type=='double' or field.type=='float':
8541 if not value.strip():
8542 value = None
8543 else:
8544 value = float(value)
8545 elif field.type in ('integer','bigint'):
8546 if not value.strip():
8547 value = None
8548 else:
8549 value = int(value)
8550 elif field.type.startswith('list:string'):
8551 value = bar_decode_string(value)
8552 elif field.type.startswith(list_reference_s):
8553 ref_table = field.type[len(list_reference_s):].strip()
8554 if id_map is not None:
8555 value = [id_map[ref_table][int(v)] \
8556 for v in bar_decode_string(value)]
8557 else:
8558 value = [v for v in bar_decode_string(value)]
8559 elif field.type.startswith('list:'):
8560 value = bar_decode_integer(value)
8561 elif id_map and field.type.startswith('reference'):
8562 try:
8563 value = id_map[field.type[9:].strip()][int(value)]
8564 except KeyError:
8565 pass
8566 elif id_offset and field.type.startswith('reference'):
8567 try:
8568 value = id_offset[field.type[9:].strip()]+int(value)
8569 except KeyError:
8570 pass
8571 return (field.name, value)
8572
8573 def is_id(colname):
8574 if colname in self:
8575 return self[colname].type == 'id'
8576 else:
8577 return False
8578
8579 first = True
8580 unique_idx = None
8581 for line in reader:
8582 if not line:
8583 break
8584 if not colnames:
8585 colnames = [x.split('.',1)[-1] for x in line][:len(line)]
8586 cols, cid = [], None
8587 for i,colname in enumerate(colnames):
8588 if is_id(colname):
8589 cid = i
8590 else:
8591 cols.append(i)
8592 if colname == unique:
8593 unique_idx = i
8594 else:
8595 items = [fix(self[colnames[i]], line[i], id_map, id_offset) \
8596 for i in cols if colnames[i] in self.fields]
8597
8598 if not id_map and cid is not None and id_offset is not None and not unique_idx:
8599 csv_id = int(line[cid])
8600 curr_id = self.insert(**dict(items))
8601 if first:
8602 first = False
8603
8604
8605
8606 if curr_id>csv_id:
8607 id_offset[self._tablename] = curr_id-csv_id
8608 else:
8609 id_offset[self._tablename] = 0
8610
8611 while curr_id<csv_id+id_offset[self._tablename]:
8612 self._db(self._db[self][colnames[cid]] == curr_id).delete()
8613 curr_id = self.insert(**dict(items))
8614
8615
8616 elif not unique_idx:
8617 new_id = self.insert(**dict(items))
8618 else:
8619 unique_value = line[unique_idx]
8620 query = self._db[self][unique] == unique_value
8621 record = self._db(query).select().first()
8622 if record:
8623 record.update_record(**dict(items))
8624 new_id = record[self._id.name]
8625 else:
8626 new_id = self.insert(**dict(items))
8627 if id_map and cid is not None:
8628 id_map_self[int(line[cid])] = new_id
8629
8630 - def as_dict(self, flat=False, sanitize=True, field_options=True):
8631 tablename = str(self)
8632 table_as_dict = dict(name=tablename, items={}, fields=[],
8633 sequence_name=self._sequence_name,
8634 trigger_name=self._trigger_name,
8635 common_filter=self._common_filter, format=self._format,
8636 singular=self._singular, plural=self._plural)
8637
8638 for field in self:
8639 if (field.readable or field.writable) or (not sanitize):
8640 table_as_dict["fields"].append(field.name)
8641 table_as_dict["items"][field.name] = \
8642 field.as_dict(flat=flat, sanitize=sanitize,
8643 options=field_options)
8644 return table_as_dict
8645
8646 - def as_xml(self, sanitize=True, field_options=True):
8652
8653 - def as_json(self, sanitize=True, field_options=True):
8659
8660 - def as_yaml(self, sanitize=True, field_options=True):
8666
8669
8670 - def on(self, query):
8671 return Expression(self._db,self._db._adapter.ON,self,query)
8672
8674 tablenames = qset.db._adapter.tables(qset.query)
8675 if len(tablenames)!=1: raise RuntimeError("cannot update join")
8676 table = qset.db[tablenames[0]]
8677 for row in qset.select():
8678 fields = archive_table._filter_fields(row)
8679 fields[current_record] = row.id
8680 archive_table.insert(**fields)
8681 return False
8682
8686
8687 - def __init__(
8688 self,
8689 db,
8690 op,
8691 first=None,
8692 second=None,
8693 type=None,
8694 **optional_args
8695 ):
8696
8697 self.db = db
8698 self.op = op
8699 self.first = first
8700 self.second = second
8701 self._table = getattr(first,'_table',None)
8702
8703 if not type and first and hasattr(first,'type'):
8704 self.type = first.type
8705 else:
8706 self.type = type
8707 self.optional_args = optional_args
8708
8712
8716
8720
8724
8728
8732
8736
8740
8744
8748
8752
8756
8760
8764
8768
8772
8776
8778 db = self.db
8779 if start < 0:
8780 pos0 = '(%s - %d)' % (self.len(), abs(start) - 1)
8781 else:
8782 pos0 = start + 1
8783
8784 if stop < 0:
8785 length = '(%s - %d - %s)' % (self.len(), abs(stop) - 1, pos0)
8786 elif stop == sys.maxint:
8787 length = self.len()
8788 else:
8789 length = '(%s - %s)' % (stop + 1, pos0)
8790 return Expression(db,db._adapter.SUBSTRING,
8791 self, (pos0, length), self.type)
8792
8794 return self[i:i + 1]
8795
8797 return self.db._adapter.expand(self,self.type)
8798
8800 db = self.db
8801 return Expression(db,db._adapter.COMMA,self,other,self.type)
8802
8808
8812
8814 db = self.db
8815 if self.type in ('integer','bigint'):
8816 result_type = 'integer'
8817 elif self.type in ['date','time','datetime','double','float']:
8818 result_type = 'double'
8819 else:
8820 raise SyntaxError("subtraction operation not supported for type")
8821 return Expression(db,db._adapter.SUB,self,other,result_type)
8822
8826
8830
8834
8838
8842
8846
8850
8854
8858
8859 - def like(self, value, case_sensitive=False):
8860 db = self.db
8861 op = case_sensitive and db._adapter.LIKE or db._adapter.ILIKE
8862 return Query(db, op, self, value)
8863
8867
8869 """
8870 Accepts the following inputs:
8871 field.belongs(1,2)
8872 field.belongs((1,2))
8873 field.belongs(query)
8874
8875 Does NOT accept:
8876 field.belongs(1)
8877 """
8878 db = self.db
8879 if len(value) == 1:
8880 value = value[0]
8881 if isinstance(value,Query):
8882 value = db(value)._select(value.first._table._id)
8883 return Query(db, db._adapter.BELONGS, self, value)
8884
8886 db = self.db
8887 if not self.type in ('string', 'text', 'json'):
8888 raise SyntaxError("startswith used with incompatible field type")
8889 return Query(db, db._adapter.STARTSWITH, self, value)
8890
8892 db = self.db
8893 if not self.type in ('string', 'text', 'json'):
8894 raise SyntaxError("endswith used with incompatible field type")
8895 return Query(db, db._adapter.ENDSWITH, self, value)
8896
8897 - def contains(self, value, all=False, case_sensitive=False):
8898 """
8899 The case_sensitive parameters is only useful for PostgreSQL
8900 For other RDMBs it is ignored and contains is always case in-sensitive
8901 For MongoDB and GAE contains is always case sensitive
8902 """
8903 db = self.db
8904 if isinstance(value,(list, tuple)):
8905 subqueries = [self.contains(str(v).strip(),case_sensitive=case_sensitive)
8906 for v in value if str(v).strip()]
8907 if not subqueries:
8908 return self.contains('')
8909 else:
8910 return reduce(all and AND or OR,subqueries)
8911 if not self.type in ('string', 'text', 'json') and not self.type.startswith('list:'):
8912 raise SyntaxError("contains used with incompatible field type")
8913 return Query(db, db._adapter.CONTAINS, self, value, case_sensitive=case_sensitive)
8914
8918
8919
8920
8921 - def st_asgeojson(self, precision=15, options=0, version=1):
8925
8926 - def st_astext(self):
8927 db = self.db
8928 return Expression(db, db._adapter.ST_ASTEXT, self, type='string')
8929
8933
8937
8941
8945
8946
8947
8951
8955
8959
8963
8967
8971
8976 """
8977 allows defining of custom SQL types
8978
8979 Example::
8980
8981 decimal = SQLCustomType(
8982 type ='double',
8983 native ='integer',
8984 encoder =(lambda x: int(float(x) * 100)),
8985 decoder = (lambda x: Decimal("0.00") + Decimal(str(float(x)/100)) )
8986 )
8987
8988 db.define_table(
8989 'example',
8990 Field('value', type=decimal)
8991 )
8992
8993 :param type: the web2py type (default = 'string')
8994 :param native: the backend type
8995 :param encoder: how to encode the value to store it in the backend
8996 :param decoder: how to decode the value retrieved from the backend
8997 :param validator: what validators to use ( default = None, will use the
8998 default validator for type)
8999 """
9000
9001 - def __init__(
9002 self,
9003 type='string',
9004 native=None,
9005 encoder=None,
9006 decoder=None,
9007 validator=None,
9008 _class=None,
9009 ):
9010
9011 self.type = type
9012 self.native = native
9013 self.encoder = encoder or (lambda x: x)
9014 self.decoder = decoder or (lambda x: x)
9015 self.validator = validator
9016 self._class = _class or type
9017
9019 try:
9020 return self.type.startswith(self, text)
9021 except TypeError:
9022 return False
9023
9026
9029
9032
9034 - def __init__(self, name, f=None, ftype='string',label=None,table_name=None):
9048
9050 - def __init__(self, name, f=None, handler=None):
9054
9056 return ', '.join(str(y) for y in x or [])
9057
9058 -class Field(Expression):
9059
9060 Virtual = FieldVirtual
9061 Method = FieldMethod
9062 Lazy = FieldMethod
9063
9064 """
9065 an instance of this class represents a database field
9066
9067 example::
9068
9069 a = Field(name, 'string', length=32, default=None, required=False,
9070 requires=IS_NOT_EMPTY(), ondelete='CASCADE',
9071 notnull=False, unique=False,
9072 uploadfield=True, widget=None, label=None, comment=None,
9073 uploadfield=True, # True means store on disk,
9074 # 'a_field_name' means store in this field in db
9075 # False means file content will be discarded.
9076 writable=True, readable=True, update=None, authorize=None,
9077 autodelete=False, represent=None, uploadfolder=None,
9078 uploadseparate=False # upload to separate directories by uuid_keys
9079 # first 2 character and tablename.fieldname
9080 # False - old behavior
9081 # True - put uploaded file in
9082 # <uploaddir>/<tablename>.<fieldname>/uuid_key[:2]
9083 # directory)
9084 uploadfs=None # a pyfilesystem where to store upload
9085
9086 to be used as argument of DAL.define_table
9087
9088 allowed field types:
9089 string, boolean, integer, double, text, blob,
9090 date, time, datetime, upload, password
9091
9092 strings must have a length of Adapter.maxcharlength by default (512 or 255 for mysql)
9093 fields should have a default or they will be required in SQLFORMs
9094 the requires argument is used to validate the field input in SQLFORMs
9095
9096 """
9097
9098 - def __init__(
9099 self,
9100 fieldname,
9101 type='string',
9102 length=None,
9103 default=DEFAULT,
9104 required=False,
9105 requires=DEFAULT,
9106 ondelete='CASCADE',
9107 notnull=False,
9108 unique=False,
9109 uploadfield=True,
9110 widget=None,
9111 label=None,
9112 comment=None,
9113 writable=True,
9114 readable=True,
9115 update=None,
9116 authorize=None,
9117 autodelete=False,
9118 represent=None,
9119 uploadfolder=None,
9120 uploadseparate=False,
9121 uploadfs=None,
9122 compute=None,
9123 custom_store=None,
9124 custom_retrieve=None,
9125 custom_retrieve_file_properties=None,
9126 custom_delete=None,
9127 filter_in = None,
9128 filter_out = None,
9129 custom_qualifier = None,
9130 map_none = None,
9131 ):
9132 self._db = self.db = None
9133 self.op = None
9134 self.first = None
9135 self.second = None
9136 self.name = fieldname = cleanup(fieldname)
9137 if not isinstance(fieldname,str) or hasattr(Table,fieldname) or \
9138 fieldname[0] == '_' or REGEX_PYTHON_KEYWORDS.match(fieldname):
9139 raise SyntaxError('Field: invalid field name: %s' % fieldname)
9140 self.type = type if not isinstance(type, (Table,Field)) else 'reference %s' % type
9141 self.length = length if not length is None else DEFAULTLENGTH.get(self.type,512)
9142 self.default = default if default!=DEFAULT else (update or None)
9143 self.required = required
9144 self.ondelete = ondelete.upper()
9145 self.notnull = notnull
9146 self.unique = unique
9147 self.uploadfield = uploadfield
9148 self.uploadfolder = uploadfolder
9149 self.uploadseparate = uploadseparate
9150 self.uploadfs = uploadfs
9151 self.widget = widget
9152 self.comment = comment
9153 self.writable = writable
9154 self.readable = readable
9155 self.update = update
9156 self.authorize = authorize
9157 self.autodelete = autodelete
9158 self.represent = list_represent if \
9159 represent==None and type in ('list:integer','list:string') else represent
9160 self.compute = compute
9161 self.isattachment = True
9162 self.custom_store = custom_store
9163 self.custom_retrieve = custom_retrieve
9164 self.custom_retrieve_file_properties = custom_retrieve_file_properties
9165 self.custom_delete = custom_delete
9166 self.filter_in = filter_in
9167 self.filter_out = filter_out
9168 self.custom_qualifier = custom_qualifier
9169 self.label = label if label!=None else fieldname.replace('_',' ').title()
9170 self.requires = requires if requires!=None else []
9171 self.map_none = map_none
9172
9174 self.__dict__.update(*args,**attributes)
9175
9176 - def clone(self,point_self_references_to=False,**args):
9177 field = copy.copy(self)
9178 if point_self_references_to and \
9179 field.type == 'reference %s'+field._tablename:
9180 field.type = 'reference %s' % point_self_references_to
9181 field.__dict__.update(args)
9182 return field
9183
9184 - def store(self, file, filename=None, path=None):
9185 if self.custom_store:
9186 return self.custom_store(file,filename,path)
9187 if isinstance(file, cgi.FieldStorage):
9188 filename = filename or file.filename
9189 file = file.file
9190 elif not filename:
9191 filename = file.name
9192 filename = os.path.basename(filename.replace('/', os.sep)\
9193 .replace('\\', os.sep))
9194 m = REGEX_STORE_PATTERN.search(filename)
9195 extension = m and m.group('e') or 'txt'
9196 uuid_key = web2py_uuid().replace('-', '')[-16:]
9197 encoded_filename = base64.b16encode(filename).lower()
9198 newfilename = '%s.%s.%s.%s' % \
9199 (self._tablename, self.name, uuid_key, encoded_filename)
9200 newfilename = newfilename[:(self.length - 1 - len(extension))] + '.' + extension
9201 self_uploadfield = self.uploadfield
9202 if isinstance(self_uploadfield,Field):
9203 blob_uploadfield_name = self_uploadfield.uploadfield
9204 keys={self_uploadfield.name: newfilename,
9205 blob_uploadfield_name: file.read()}
9206 self_uploadfield.table.insert(**keys)
9207 elif self_uploadfield == True:
9208 if path:
9209 pass
9210 elif self.uploadfolder:
9211 path = self.uploadfolder
9212 elif self.db._adapter.folder:
9213 path = pjoin(self.db._adapter.folder, '..', 'uploads')
9214 else:
9215 raise RuntimeError(
9216 "you must specify a Field(...,uploadfolder=...)")
9217 if self.uploadseparate:
9218 if self.uploadfs:
9219 raise RuntimeError("not supported")
9220 path = pjoin(path,"%s.%s" %(self._tablename, self.name),
9221 uuid_key[:2])
9222 if not exists(path):
9223 os.makedirs(path)
9224 pathfilename = pjoin(path, newfilename)
9225 if self.uploadfs:
9226 dest_file = self.uploadfs.open(newfilename, 'wb')
9227 else:
9228 dest_file = open(pathfilename, 'wb')
9229 try:
9230 shutil.copyfileobj(file, dest_file)
9231 except IOError:
9232 raise IOError(
9233 'Unable to store file "%s" because invalid permissions, readonly file system, or filename too long' % pathfilename)
9234 dest_file.close()
9235 return newfilename
9236
9238 self_uploadfield = self.uploadfield
9239 if self.custom_retrieve:
9240 return self.custom_retrieve(name, path)
9241 import http
9242 if self.authorize or isinstance(self_uploadfield, str):
9243 row = self.db(self == name).select().first()
9244 if not row:
9245 raise http.HTTP(404)
9246 if self.authorize and not self.authorize(row):
9247 raise http.HTTP(403)
9248 m = REGEX_UPLOAD_PATTERN.match(name)
9249 if not m or not self.isattachment:
9250 raise TypeError('Can\'t retrieve %s' % name)
9251 file_properties = self.retrieve_file_properties(name,path)
9252 filename = file_properties['filename']
9253 if isinstance(self_uploadfield, str):
9254 stream = StringIO.StringIO(row[self_uploadfield] or '')
9255 elif isinstance(self_uploadfield,Field):
9256 blob_uploadfield_name = self_uploadfield.uploadfield
9257 query = self_uploadfield == name
9258 data = self_uploadfield.table(query)[blob_uploadfield_name]
9259 stream = StringIO.StringIO(data)
9260 elif self.uploadfs:
9261
9262 stream = self.uploadfs.open(name, 'rb')
9263 else:
9264
9265 stream = pjoin(file_properties['path'], name)
9266 return (filename, stream)
9267
9269 self_uploadfield = self.uploadfield
9270 if self.custom_retrieve_file_properties:
9271 return self.custom_retrieve_file_properties(name, path)
9272 try:
9273 m = REGEX_UPLOAD_PATTERN.match(name)
9274 if not m or not self.isattachment:
9275 raise TypeError('Can\'t retrieve %s file properties' % name)
9276 filename = base64.b16decode(m.group('name'), True)
9277 filename = REGEX_CLEANUP_FN.sub('_', filename)
9278 except (TypeError, AttributeError):
9279 filename = name
9280 if isinstance(self_uploadfield, str):
9281 return dict(path=None,filename=filename)
9282 elif isinstance(self_uploadfield,Field):
9283 return dict(path=None,filename=filename)
9284 else:
9285
9286 if path:
9287 pass
9288 elif self.uploadfolder:
9289 path = self.uploadfolder
9290 else:
9291 path = pjoin(self.db._adapter.folder, '..', 'uploads')
9292 if self.uploadseparate:
9293 t = m.group('table')
9294 f = m.group('field')
9295 u = m.group('uuidkey')
9296 path = pjoin(path,"%s.%s" % (t,f),u[:2])
9297 return dict(path=path,filename=filename)
9298
9299
9315
9327
9328 - def count(self, distinct=None):
9330
9331 - def as_dict(self, flat=False, sanitize=True, options=True):
9332
9333 attrs = ('type', 'length', 'default', 'required',
9334 'ondelete', 'notnull', 'unique', 'uploadfield',
9335 'widget', 'label', 'comment', 'writable', 'readable',
9336 'update', 'authorize', 'autodelete', 'represent',
9337 'uploadfolder', 'uploadseparate', 'uploadfs',
9338 'compute', 'custom_store', 'custom_retrieve',
9339 'custom_retrieve_file_properties', 'custom_delete',
9340 'filter_in', 'filter_out', 'custom_qualifier',
9341 'map_none', 'name')
9342
9343 SERIALIZABLE_TYPES = (int, long, basestring, dict, list,
9344 float, tuple, bool, type(None))
9345
9346 def flatten(obj):
9347 if flat:
9348 if isinstance(obj, flatten.__class__):
9349 return str(type(obj))
9350 elif isinstance(obj, type):
9351 try:
9352 return str(obj).split("'")[1]
9353 except IndexError:
9354 return str(obj)
9355 elif not isinstance(obj, SERIALIZABLE_TYPES):
9356 return str(obj)
9357 elif isinstance(obj, dict):
9358 newobj = dict()
9359 for k, v in obj.items():
9360 newobj[k] = flatten(v)
9361 return newobj
9362 elif isinstance(obj, (list, tuple, set)):
9363 return [flatten(v) for v in obj]
9364 else:
9365 return obj
9366 elif isinstance(obj, (dict, set)):
9367 return obj.copy()
9368 else: return obj
9369
9370 def filter_requires(t, r, options=True):
9371 if sanitize and any([keyword in str(t).upper() for
9372 keyword in ("CRYPT", "IS_STRONG")]):
9373 return None
9374
9375 if not isinstance(r, dict):
9376 if options and hasattr(r, "options"):
9377 if callable(r.options):
9378 r.options()
9379 newr = r.__dict__.copy()
9380 else:
9381 newr = r.copy()
9382
9383
9384 if not options and newr.has_key("labels"):
9385 [newr.update({key:None}) for key in
9386 ("labels", "theset") if (key in newr)]
9387
9388 for k, v in newr.items():
9389 if k == "other":
9390 if isinstance(v, dict):
9391 otype, other = v.popitem()
9392 else:
9393 otype = flatten(type(v))
9394 other = v
9395 newr[k] = {otype: filter_requires(otype, other,
9396 options=options)}
9397 else:
9398 newr[k] = flatten(v)
9399 return newr
9400
9401 if isinstance(self.requires, (tuple, list, set)):
9402 requires = dict([(flatten(type(r)),
9403 filter_requires(type(r), r,
9404 options=options)) for
9405 r in self.requires])
9406 else:
9407 requires = {flatten(type(self.requires)):
9408 filter_requires(type(self.requires),
9409 self.requires, options=options)}
9410
9411 d = dict(colname="%s.%s" % (self.tablename, self.name),
9412 requires=requires)
9413 d.update([(attr, flatten(getattr(self, attr))) for attr in attrs])
9414 return d
9415
9416 - def as_xml(self, sanitize=True, options=True):
9424
9425 - def as_json(self, sanitize=True, options=True):
9433
9434 - def as_yaml(self, sanitize=True, options=True):
9441
9444
9446 try:
9447 return '%s.%s' % (self.tablename, self.name)
9448 except:
9449 return '<no table>.%s' % self.name
9450
9451
9452 -class Query(object):
9453
9454 """
9455 a query object necessary to define a set.
9456 it can be stored or can be passed to DAL.__call__() to obtain a Set
9457
9458 Example::
9459
9460 query = db.users.name=='Max'
9461 set = db(query)
9462 records = set.select()
9463
9464 """
9465
9466 - def __init__(
9467 self,
9468 db,
9469 op,
9470 first=None,
9471 second=None,
9472 ignore_common_filters = False,
9473 **optional_args
9474 ):
9475 self.db = self._db = db
9476 self.op = op
9477 self.first = first
9478 self.second = second
9479 self.ignore_common_filters = ignore_common_filters
9480 self.optional_args = optional_args
9481
9484
9486 return self.db._adapter.expand(self)
9487
9490
9492 return Query(self.db,self.db._adapter.OR,self,other)
9493
9495 if self.op==self.db._adapter.NOT:
9496 return self.first
9497 return Query(self.db,self.db._adapter.NOT,self)
9498
9500 return repr(self) == repr(other)
9501
9503 return not (self == other)
9504
9505 - def case(self,t=1,f=0):
9506 return self.db._adapter.CASE(self,t,f)
9507
9508 - def as_dict(self, flat=False, sanitize=True):
9509 """Experimental stuff
9510
9511 This allows to return a plain dictionary with the basic
9512 query representation. Can be used with json/xml services
9513 for client-side db I/O
9514
9515 Example:
9516 >>> q = db.auth_user.id != 0
9517 >>> q.as_dict(flat=True)
9518 {"op": "NE", "first":{"tablename": "auth_user",
9519 "fieldname": "id"},
9520 "second":0}
9521 """
9522
9523 SERIALIZABLE_TYPES = (tuple, dict, list, int, long, float,
9524 basestring, type(None), bool)
9525 def loop(d):
9526 newd = dict()
9527 for k, v in d.items():
9528 if k in ("first", "second"):
9529 if isinstance(v, self.__class__):
9530 newd[k] = loop(v.__dict__)
9531 elif isinstance(v, Field):
9532 newd[k] = {"tablename": v._tablename,
9533 "fieldname": v.name}
9534 elif isinstance(v, Expression):
9535 newd[k] = loop(v.__dict__)
9536 elif isinstance(v, SERIALIZABLE_TYPES):
9537 newd[k] = v
9538 elif k == "op":
9539 if callable(v):
9540 newd[k] = v.__name__
9541 elif isinstance(v, basestring):
9542 newd[k] = v
9543 else: pass
9544 elif isinstance(v, SERIALIZABLE_TYPES):
9545 if isinstance(v, dict):
9546 newd[k] = loop(v)
9547 else: newd[k] = v
9548 return newd
9549
9550 if flat:
9551 return loop(self.__dict__)
9552 else: return self.__dict__
9553
9554
9555 - def as_xml(self, sanitize=True):
9562
9563 - def as_json(self, sanitize=True):
9570
9572 if not orderby:
9573 return None
9574 orderby2 = orderby[0]
9575 for item in orderby[1:]:
9576 orderby2 = orderby2 | item
9577 return orderby2
9578
9580 return (query and hasattr(query,'ignore_common_filters') and \
9581 not query.ignore_common_filters)
9582
9584
9585 """
9586 a Set represents a set of records in the database,
9587 the records are identified by the query=Query(...) object.
9588 normally the Set is generated by DAL.__call__(Query(...))
9589
9590 given a set, for example
9591 set = db(db.users.name=='Max')
9592 you can:
9593 set.update(db.users.name='Massimo')
9594 set.delete() # all elements in the set
9595 set.select(orderby=db.users.id, groupby=db.users.name, limitby=(0,10))
9596 and take subsets:
9597 subset = set(db.users.id<5)
9598 """
9599
9600 - def __init__(self, db, query, ignore_common_filters = None):
9601 self.db = db
9602 self._db = db
9603 self.dquery = None
9604
9605
9606 if isinstance(query, dict):
9607 query = self.parse(query)
9608
9609 if not ignore_common_filters is None and \
9610 use_common_filters(query) == ignore_common_filters:
9611 query = copy.copy(query)
9612 query.ignore_common_filters = ignore_common_filters
9613 self.query = query
9614
9617
9618 - def __call__(self, query, ignore_common_filters=False):
9619 if isinstance(query,Table):
9620 query = self.db._adapter.id_query(query)
9621 elif isinstance(query,str):
9622 query = Expression(self.db,query)
9623 elif isinstance(query,Field):
9624 query = query!=None
9625 if self.query:
9626 return Set(self.db, self.query & query,
9627 ignore_common_filters=ignore_common_filters)
9628 else:
9629 return Set(self.db, query,
9630 ignore_common_filters=ignore_common_filters)
9631
9632 - def _count(self,distinct=None):
9633 return self.db._adapter._count(self.query,distinct)
9634
9635 - def _select(self, *fields, **attributes):
9636 adapter = self.db._adapter
9637 tablenames = adapter.tables(self.query,
9638 attributes.get('join',None),
9639 attributes.get('left',None),
9640 attributes.get('orderby',None),
9641 attributes.get('groupby',None))
9642 fields = adapter.expand_all(fields, tablenames)
9643 return adapter._select(self.query,fields,attributes)
9644
9646 db = self.db
9647 tablename = db._adapter.get_table(self.query)
9648 return db._adapter._delete(tablename,self.query)
9649
9650 - def _update(self, **update_fields):
9655
9656 - def as_dict(self, flat=False, sanitize=True):
9657 if flat:
9658 uid = dbname = uri = None
9659 codec = self.db._db_codec
9660 if not sanitize:
9661 uri, dbname, uid = (self.db._dbname, str(self.db),
9662 self.db._db_uid)
9663 d = {"query": self.query.as_dict(flat=flat)}
9664 d["db"] = {"uid": uid, "codec": codec,
9665 "name": dbname, "uri": uri}
9666 return d
9667 else: return self.__dict__
9668
9669 - def as_xml(self, sanitize=True):
9676
9677 - def as_json(self, sanitize=True):
9684
9685 - def parse(self, dquery):
9686 "Experimental: Turn a dictionary into a Query object"
9687 self.dquery = dquery
9688 return self.build(self.dquery)
9689
9691 "Experimental: see .parse()"
9692 op, first, second = (d["op"], d["first"],
9693 d.get("second", None))
9694 left = right = built = None
9695
9696 if op in ("AND", "OR"):
9697 if not (type(first), type(second)) == (dict, dict):
9698 raise SyntaxError("Invalid AND/OR query")
9699 if op == "AND":
9700 built = self.build(first) & self.build(second)
9701 else: built = self.build(first) | self.build(second)
9702
9703 elif op == "NOT":
9704 if first is None:
9705 raise SyntaxError("Invalid NOT query")
9706 built = ~self.build(first)
9707 else:
9708
9709 for k, v in {"left": first, "right": second}.items():
9710 if isinstance(v, dict) and v.get("op"):
9711 v = self.build(v)
9712 if isinstance(v, dict) and ("tablename" in v):
9713 v = self.db[v["tablename"]][v["fieldname"]]
9714 if k == "left": left = v
9715 else: right = v
9716
9717 if hasattr(self.db._adapter, op):
9718 opm = getattr(self.db._adapter, op)
9719
9720 if op == "EQ": built = left == right
9721 elif op == "NE": built = left != right
9722 elif op == "GT": built = left > right
9723 elif op == "GE": built = left >= right
9724 elif op == "LT": built = left < right
9725 elif op == "LE": built = left <= right
9726 elif op in ("JOIN", "LEFT_JOIN", "RANDOM", "ALLOW_NULL"):
9727 built = Expression(self.db, opm)
9728 elif op in ("LOWER", "UPPER", "EPOCH", "PRIMARY_KEY",
9729 "COALESCE_ZERO", "RAW", "INVERT"):
9730 built = Expression(self.db, opm, left)
9731 elif op in ("COUNT", "EXTRACT", "AGGREGATE", "SUBSTRING",
9732 "REGEXP", "LIKE", "ILIKE", "STARTSWITH",
9733 "ENDSWITH", "ADD", "SUB", "MUL", "DIV",
9734 "MOD", "AS", "ON", "COMMA", "NOT_NULL",
9735 "COALESCE", "CONTAINS", "BELONGS"):
9736 built = Expression(self.db, opm, left, right)
9737
9738 elif not (left or right): built = Expression(self.db, op)
9739 else:
9740 raise SyntaxError("Operator not supported: %s" % op)
9741
9742 return built
9743
9745 return not self.select(limitby=(0,1))
9746
9747 - def count(self,distinct=None, cache=None):
9748 db = self.db
9749 if cache:
9750 cache_model, time_expire = cache
9751 sql = self._count(distinct=distinct)
9752 key = db._uri + '/' + sql
9753 if len(key)>200: key = hashlib_md5(key).hexdigest()
9754 return cache_model(
9755 key,
9756 (lambda self=self,distinct=distinct: \
9757 db._adapter.count(self.query,distinct)),
9758 time_expire)
9759 return db._adapter.count(self.query,distinct)
9760
9761 - def select(self, *fields, **attributes):
9762 adapter = self.db._adapter
9763 tablenames = adapter.tables(self.query,
9764 attributes.get('join',None),
9765 attributes.get('left',None),
9766 attributes.get('orderby',None),
9767 attributes.get('groupby',None))
9768 fields = adapter.expand_all(fields, tablenames)
9769 return adapter.select(self.query,fields,attributes)
9770
9773
9775 db = self.db
9776 tablename = db._adapter.get_table(self.query)
9777 table = db[tablename]
9778 if any(f(self) for f in table._before_delete): return 0
9779 ret = db._adapter.delete(tablename,self.query)
9780 ret and [f(self) for f in table._after_delete]
9781 return ret
9782
9783 - def update(self, **update_fields):
9784 db = self.db
9785 tablename = db._adapter.get_table(self.query)
9786 table = db[tablename]
9787 table._attempt_upload(update_fields)
9788 if any(f(self,update_fields) for f in table._before_update):
9789 return 0
9790 fields = table._listify(update_fields,update=True)
9791 if not fields:
9792 raise SyntaxError("No fields to update")
9793 ret = db._adapter.update(tablename,self.query,fields)
9794 ret and [f(self,update_fields) for f in table._after_update]
9795 return ret
9796
9798 """
9799 same as update but does not call table._before_update and _after_update
9800 """
9801 tablename = self.db._adapter.get_table(self.query)
9802 table = self.db[tablename]
9803 fields = table._listify(update_fields,update=True)
9804 if not fields: raise SyntaxError("No fields to update")
9805 ret = self.db._adapter.update(tablename,self.query,fields)
9806 return ret
9807
9809 tablename = self.db._adapter.get_table(self.query)
9810 response = Row()
9811 response.errors = Row()
9812 new_fields = copy.copy(update_fields)
9813 for key,value in update_fields.iteritems():
9814 value,error = self.db[tablename][key].validate(value)
9815 if error:
9816 response.errors[key] = error
9817 else:
9818 new_fields[key] = value
9819 table = self.db[tablename]
9820 if response.errors:
9821 response.updated = None
9822 else:
9823 if not any(f(self,new_fields) for f in table._before_update):
9824 fields = table._listify(new_fields,update=True)
9825 if not fields: raise SyntaxError("No fields to update")
9826 ret = self.db._adapter.update(tablename,self.query,fields)
9827 ret and [f(self,new_fields) for f in table._after_update]
9828 else:
9829 ret = 0
9830 response.updated = ret
9831 return response
9832
9834 table = self.db[self.db._adapter.tables(self.query)[0]]
9835
9836 if upload_fields:
9837 fields = upload_fields.keys()
9838 else:
9839 fields = table.fields
9840 fields = [f for f in fields if table[f].type == 'upload'
9841 and table[f].uploadfield == True
9842 and table[f].autodelete]
9843 if not fields:
9844 return False
9845 for record in self.select(*[table[f] for f in fields]):
9846 for fieldname in fields:
9847 field = table[fieldname]
9848 oldname = record.get(fieldname, None)
9849 if not oldname:
9850 continue
9851 if upload_fields and oldname == upload_fields[fieldname]:
9852 continue
9853 if field.custom_delete:
9854 field.custom_delete(oldname)
9855 else:
9856 uploadfolder = field.uploadfolder
9857 if not uploadfolder:
9858 uploadfolder = pjoin(
9859 self.db._adapter.folder, '..', 'uploads')
9860 if field.uploadseparate:
9861 items = oldname.split('.')
9862 uploadfolder = pjoin(
9863 uploadfolder,
9864 "%s.%s" % (items[0], items[1]),
9865 items[2][:2])
9866 oldpath = pjoin(uploadfolder, oldname)
9867 if exists(oldpath):
9868 os.unlink(oldpath)
9869 return False
9870
9872 - def __init__(self, colset, table, id):
9873 self.colset, self.db, self.tablename, self.id = \
9874 colset, table._db, table._tablename, id
9875
9877 colset, db, tablename, id = self.colset, self.db, self.tablename, self.id
9878 table = db[tablename]
9879 newfields = fields or dict(colset)
9880 for fieldname in newfields.keys():
9881 if not fieldname in table.fields or table[fieldname].type=='id':
9882 del newfields[fieldname]
9883 table._db(table._id==id,ignore_common_filters=True).update(**newfields)
9884 colset.update(newfields)
9885 return colset
9886
9889 self.db, self.tablename, self.id = table._db, table._tablename, id
9891 return self.db(self.db[self.tablename]._id==self.id).delete()
9892
9895 self.db, self.tablename, self.fieldname, self.id = \
9896 field.db, field._tablename, field.name, id
9898 query = self.db[self.tablename][self.fieldname]==self.id
9899 return Set(self.db,query)
9902 - def __call__(self, query, ignore_common_filters=False):
9903 return self._getset()(query, ignore_common_filters)
9904 - def _count(self,distinct=None):
9906 - def _select(self, *fields, **attributes):
9910 - def _update(self, **update_fields):
9914 - def count(self,distinct=None, cache=None):
9916 - def select(self, *fields, **attributes):
9922 - def update(self, **update_fields):
9930
9933 self.method=method
9934 self.row=row
9936 return self.method(self.row,*args,**kwargs)
9937
9939 f.__lazy__ = True
9940 return f
9941
9942 -class Rows(object):
9943
9944 """
9945 A wrapper for the return value of a select. It basically represents a table.
9946 It has an iterator and each row is represented as a dictionary.
9947 """
9948
9949
9950
9951 - def __init__(
9952 self,
9953 db=None,
9954 records=[],
9955 colnames=[],
9956 compact=True,
9957 rawrows=None
9958 ):
9959 self.db = db
9960 self.records = records
9961 self.colnames = colnames
9962 self.compact = compact
9963 self.response = rawrows
9964
9966 return '<Rows (%s)>' % len(self.records)
9967
9969 """
9970 db.define_table('x',Field('number','integer'))
9971 if db(db.x).isempty(): [db.x.insert(number=i) for i in range(10)]
9972
9973 from gluon.dal import lazy_virtualfield
9974
9975 class MyVirtualFields(object):
9976 # normal virtual field (backward compatible, discouraged)
9977 def normal_shift(self): return self.x.number+1
9978 # lazy virtual field (because of @staticmethod)
9979 @lazy_virtualfield
9980 def lazy_shift(instance,row,delta=4): return row.x.number+delta
9981 db.x.virtualfields.append(MyVirtualFields())
9982
9983 for row in db(db.x).select():
9984 print row.number, row.normal_shift, row.lazy_shift(delta=7)
9985 """
9986 if not keyed_virtualfields:
9987 return self
9988 for row in self.records:
9989 for (tablename,virtualfields) in keyed_virtualfields.iteritems():
9990 attributes = dir(virtualfields)
9991 if not tablename in row:
9992 box = row[tablename] = Row()
9993 else:
9994 box = row[tablename]
9995 updated = False
9996 for attribute in attributes:
9997 if attribute[0] != '_':
9998 method = getattr(virtualfields,attribute)
9999 if hasattr(method,'__lazy__'):
10000 box[attribute]=VirtualCommand(method,row)
10001 elif type(method)==types.MethodType:
10002 if not updated:
10003 virtualfields.__dict__.update(row)
10004 updated = True
10005 box[attribute]=method()
10006 return self
10007
10009 if self.colnames!=other.colnames:
10010 raise Exception('Cannot & incompatible Rows objects')
10011 records = self.records+other.records
10012 return Rows(self.db,records,self.colnames)
10013
10015 if self.colnames!=other.colnames:
10016 raise Exception('Cannot | incompatible Rows objects')
10017 records = self.records
10018 records += [record for record in other.records \
10019 if not record in records]
10020 return Rows(self.db,records,self.colnames)
10021
10023 if len(self.records):
10024 return 1
10025 return 0
10026
10028 return len(self.records)
10029
10031 return Rows(self.db,self.records[a:b],self.colnames)
10032
10034 row = self.records[i]
10035 keys = row.keys()
10036 if self.compact and len(keys) == 1 and keys[0] != '_extra':
10037 return row[row.keys()[0]]
10038 return row
10039
10041 """
10042 iterator over records
10043 """
10044
10045 for i in xrange(len(self)):
10046 yield self[i]
10047
10049 """
10050 serializes the table into a csv file
10051 """
10052
10053 s = StringIO.StringIO()
10054 self.export_to_csv_file(s)
10055 return s.getvalue()
10056
10058 if not self.records:
10059 return None
10060 return self[0]
10061
10063 if not self.records:
10064 return None
10065 return self[-1]
10066
10067 - def find(self,f,limitby=None):
10068 """
10069 returns a new Rows object, a subset of the original object,
10070 filtered by the function f
10071 """
10072 if not self:
10073 return Rows(self.db, [], self.colnames)
10074 records = []
10075 if limitby:
10076 a,b = limitby
10077 else:
10078 a,b = 0,len(self)
10079 k = 0
10080 for row in self:
10081 if f(row):
10082 if a<=k: records.append(row)
10083 k += 1
10084 if k==b: break
10085 return Rows(self.db, records, self.colnames)
10086
10088 """
10089 removes elements from the calling Rows object, filtered by the function f,
10090 and returns a new Rows object containing the removed elements
10091 """
10092 if not self.records:
10093 return Rows(self.db, [], self.colnames)
10094 removed = []
10095 i=0
10096 while i<len(self):
10097 row = self[i]
10098 if f(row):
10099 removed.append(self.records[i])
10100 del self.records[i]
10101 else:
10102 i += 1
10103 return Rows(self.db, removed, self.colnames)
10104
10105 - def sort(self, f, reverse=False):
10106 """
10107 returns a list of sorted elements (not sorted in place)
10108 """
10109 rows = Rows(self.db,[],self.colnames,compact=False)
10110 rows.records = sorted(self,key=f,reverse=reverse)
10111 return rows
10112
10113
10115 """
10116 regroups the rows, by one of the fields
10117 """
10118 if not self.records:
10119 return {}
10120 key = str(field)
10121 grouped_row_group = dict()
10122
10123 for row in self:
10124 value = row[key]
10125 if not value in grouped_row_group:
10126 grouped_row_group[value] = [row]
10127 else:
10128 grouped_row_group[value].append(row)
10129 return grouped_row_group
10130
10131 - def as_list(self,
10132 compact=True,
10133 storage_to_dict=True,
10134 datetime_to_str=True,
10135 custom_types=None):
10136 """
10137 returns the data as a list or dictionary.
10138 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
10139 :param datetime_to_str: convert datetime fields as strings (default True)
10140 """
10141 (oc, self.compact) = (self.compact, compact)
10142 if storage_to_dict:
10143 items = [item.as_dict(datetime_to_str, custom_types) for item in self]
10144 else:
10145 items = [item for item in self]
10146 self.compact = compact
10147 return items
10148
10149
10150 - def as_dict(self,
10151 key='id',
10152 compact=True,
10153 storage_to_dict=True,
10154 datetime_to_str=True,
10155 custom_types=None):
10156 """
10157 returns the data as a dictionary of dictionaries (storage_to_dict=True) or records (False)
10158
10159 :param key: the name of the field to be used as dict key, normally the id
10160 :param compact: ? (default True)
10161 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
10162 :param datetime_to_str: convert datetime fields as strings (default True)
10163 """
10164
10165
10166 multi = False
10167 f = self.first()
10168 if f:
10169 multi = any([isinstance(v, f.__class__) for v in f.values()])
10170 if (not "." in key) and multi:
10171
10172 def new_key():
10173 i = 0
10174 while True:
10175 yield i
10176 i += 1
10177 key_generator = new_key()
10178 key = lambda r: key_generator.next()
10179
10180 rows = self.as_list(compact, storage_to_dict, datetime_to_str, custom_types)
10181 if isinstance(key,str) and key.count('.')==1:
10182 (table, field) = key.split('.')
10183 return dict([(r[table][field],r) for r in rows])
10184 elif isinstance(key,str):
10185 return dict([(r[key],r) for r in rows])
10186 else:
10187 return dict([(key(r),r) for r in rows])
10188
10190 """
10191 export data to csv, the first line contains the column names
10192
10193 :param ofile: where the csv must be exported to
10194 :param null: how null values must be represented (default '<NULL>')
10195 :param delimiter: delimiter to separate values (default ',')
10196 :param quotechar: character to use to quote string values (default '"')
10197 :param quoting: quote system, use csv.QUOTE_*** (default csv.QUOTE_MINIMAL)
10198 :param represent: use the fields .represent value (default False)
10199 :param colnames: list of column names to use (default self.colnames)
10200 This will only work when exporting rows objects!!!!
10201 DO NOT use this with db.export_to_csv()
10202 """
10203 delimiter = kwargs.get('delimiter', ',')
10204 quotechar = kwargs.get('quotechar', '"')
10205 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
10206 represent = kwargs.get('represent', False)
10207 writer = csv.writer(ofile, delimiter=delimiter,
10208 quotechar=quotechar, quoting=quoting)
10209 colnames = kwargs.get('colnames', self.colnames)
10210 write_colnames = kwargs.get('write_colnames',True)
10211
10212 if write_colnames:
10213 writer.writerow(colnames)
10214
10215 def none_exception(value):
10216 """
10217 returns a cleaned up value that can be used for csv export:
10218 - unicode text is encoded as such
10219 - None values are replaced with the given representation (default <NULL>)
10220 """
10221 if value is None:
10222 return null
10223 elif isinstance(value, unicode):
10224 return value.encode('utf8')
10225 elif isinstance(value,Reference):
10226 return int(value)
10227 elif hasattr(value, 'isoformat'):
10228 return value.isoformat()[:19].replace('T', ' ')
10229 elif isinstance(value, (list,tuple)):
10230 return bar_encode(value)
10231 return value
10232
10233 for record in self:
10234 row = []
10235 for col in colnames:
10236 if not REGEX_TABLE_DOT_FIELD.match(col):
10237 row.append(record._extra[col])
10238 else:
10239 (t, f) = col.split('.')
10240 field = self.db[t][f]
10241 if isinstance(record.get(t, None), (Row,dict)):
10242 value = record[t][f]
10243 else:
10244 value = record[f]
10245 if field.type=='blob' and not value is None:
10246 value = base64.b64encode(value)
10247 elif represent and field.represent:
10248 value = field.represent(value)
10249 row.append(none_exception(value))
10250 writer.writerow(row)
10251
10252 - def xml(self,strict=False,row_name='row',rows_name='rows'):
10253 """
10254 serializes the table using sqlhtml.SQLTABLE (if present)
10255 """
10256
10257 if strict:
10258 ncols = len(self.colnames)
10259 return '<%s>\n%s\n</%s>' % (rows_name,
10260 '\n'.join(row.as_xml(row_name=row_name,
10261 colnames=self.colnames) for
10262 row in self), rows_name)
10263
10264 import sqlhtml
10265 return sqlhtml.SQLTABLE(self).xml()
10266
10267 - def as_xml(self,row_name='row',rows_name='rows'):
10268 return self.xml(strict=True, row_name=row_name, rows_name=rows_name)
10269
10270 - def as_json(self, mode='object', default=None):
10288
10289
10290 as_csv = __str__
10291 json = as_json
10292
10293
10294
10295
10296
10297 -def test_all():
10298 """
10299
10300 >>> if len(sys.argv)<2: db = DAL(\"sqlite://test.db\")
10301 >>> if len(sys.argv)>1: db = DAL(sys.argv[1])
10302 >>> tmp = db.define_table('users',\
10303 Field('stringf', 'string', length=32, required=True),\
10304 Field('booleanf', 'boolean', default=False),\
10305 Field('passwordf', 'password', notnull=True),\
10306 Field('uploadf', 'upload'),\
10307 Field('blobf', 'blob'),\
10308 Field('integerf', 'integer', unique=True),\
10309 Field('doublef', 'double', unique=True,notnull=True),\
10310 Field('jsonf', 'json'),\
10311 Field('datef', 'date', default=datetime.date.today()),\
10312 Field('timef', 'time'),\
10313 Field('datetimef', 'datetime'),\
10314 migrate='test_user.table')
10315
10316 Insert a field
10317
10318 >>> db.users.insert(stringf='a', booleanf=True, passwordf='p', blobf='0A',\
10319 uploadf=None, integerf=5, doublef=3.14,\
10320 jsonf={"j": True},\
10321 datef=datetime.date(2001, 1, 1),\
10322 timef=datetime.time(12, 30, 15),\
10323 datetimef=datetime.datetime(2002, 2, 2, 12, 30, 15))
10324 1
10325
10326 Drop the table
10327
10328 >>> db.users.drop()
10329
10330 Examples of insert, select, update, delete
10331
10332 >>> tmp = db.define_table('person',\
10333 Field('name'),\
10334 Field('birth','date'),\
10335 migrate='test_person.table')
10336 >>> person_id = db.person.insert(name=\"Marco\",birth='2005-06-22')
10337 >>> person_id = db.person.insert(name=\"Massimo\",birth='1971-12-21')
10338
10339 commented len(db().select(db.person.ALL))
10340 commented 2
10341
10342 >>> me = db(db.person.id==person_id).select()[0] # test select
10343 >>> me.name
10344 'Massimo'
10345 >>> db.person[2].name
10346 'Massimo'
10347 >>> db.person(2).name
10348 'Massimo'
10349 >>> db.person(name='Massimo').name
10350 'Massimo'
10351 >>> db.person(db.person.name=='Massimo').name
10352 'Massimo'
10353 >>> row = db.person[2]
10354 >>> row.name == row['name'] == row['person.name'] == row('person.name')
10355 True
10356 >>> db(db.person.name=='Massimo').update(name='massimo') # test update
10357 1
10358 >>> db(db.person.name=='Marco').select().first().delete_record() # test delete
10359 1
10360
10361 Update a single record
10362
10363 >>> me.update_record(name=\"Max\")
10364 <Row {'name': 'Max', 'birth': datetime.date(1971, 12, 21), 'id': 2}>
10365 >>> me.name
10366 'Max'
10367
10368 Examples of complex search conditions
10369
10370 >>> len(db((db.person.name=='Max')&(db.person.birth<'2003-01-01')).select())
10371 1
10372 >>> len(db((db.person.name=='Max')&(db.person.birth<datetime.date(2003,01,01))).select())
10373 1
10374 >>> len(db((db.person.name=='Max')|(db.person.birth<'2003-01-01')).select())
10375 1
10376 >>> me = db(db.person.id==person_id).select(db.person.name)[0]
10377 >>> me.name
10378 'Max'
10379
10380 Examples of search conditions using extract from date/datetime/time
10381
10382 >>> len(db(db.person.birth.month()==12).select())
10383 1
10384 >>> len(db(db.person.birth.year()>1900).select())
10385 1
10386
10387 Example of usage of NULL
10388
10389 >>> len(db(db.person.birth==None).select()) ### test NULL
10390 0
10391 >>> len(db(db.person.birth!=None).select()) ### test NULL
10392 1
10393
10394 Examples of search conditions using lower, upper, and like
10395
10396 >>> len(db(db.person.name.upper()=='MAX').select())
10397 1
10398 >>> len(db(db.person.name.like('%ax')).select())
10399 1
10400 >>> len(db(db.person.name.upper().like('%AX')).select())
10401 1
10402 >>> len(db(~db.person.name.upper().like('%AX')).select())
10403 0
10404
10405 orderby, groupby and limitby
10406
10407 >>> people = db().select(db.person.name, orderby=db.person.name)
10408 >>> order = db.person.name|~db.person.birth
10409 >>> people = db().select(db.person.name, orderby=order)
10410
10411 >>> people = db().select(db.person.name, orderby=db.person.name, groupby=db.person.name)
10412
10413 >>> people = db().select(db.person.name, orderby=order, limitby=(0,100))
10414
10415 Example of one 2 many relation
10416
10417 >>> tmp = db.define_table('dog',\
10418 Field('name'),\
10419 Field('birth','date'),\
10420 Field('owner',db.person),\
10421 migrate='test_dog.table')
10422 >>> db.dog.insert(name='Snoopy', birth=None, owner=person_id)
10423 1
10424
10425 A simple JOIN
10426
10427 >>> len(db(db.dog.owner==db.person.id).select())
10428 1
10429
10430 >>> len(db().select(db.person.ALL, db.dog.name,left=db.dog.on(db.dog.owner==db.person.id)))
10431 1
10432
10433 Drop tables
10434
10435 >>> db.dog.drop()
10436 >>> db.person.drop()
10437
10438 Example of many 2 many relation and Set
10439
10440 >>> tmp = db.define_table('author', Field('name'),\
10441 migrate='test_author.table')
10442 >>> tmp = db.define_table('paper', Field('title'),\
10443 migrate='test_paper.table')
10444 >>> tmp = db.define_table('authorship',\
10445 Field('author_id', db.author),\
10446 Field('paper_id', db.paper),\
10447 migrate='test_authorship.table')
10448 >>> aid = db.author.insert(name='Massimo')
10449 >>> pid = db.paper.insert(title='QCD')
10450 >>> tmp = db.authorship.insert(author_id=aid, paper_id=pid)
10451
10452 Define a Set
10453
10454 >>> authored_papers = db((db.author.id==db.authorship.author_id)&(db.paper.id==db.authorship.paper_id))
10455 >>> rows = authored_papers.select(db.author.name, db.paper.title)
10456 >>> for row in rows: print row.author.name, row.paper.title
10457 Massimo QCD
10458
10459 Example of search condition using belongs
10460
10461 >>> set = (1, 2, 3)
10462 >>> rows = db(db.paper.id.belongs(set)).select(db.paper.ALL)
10463 >>> print rows[0].title
10464 QCD
10465
10466 Example of search condition using nested select
10467
10468 >>> nested_select = db()._select(db.authorship.paper_id)
10469 >>> rows = db(db.paper.id.belongs(nested_select)).select(db.paper.ALL)
10470 >>> print rows[0].title
10471 QCD
10472
10473 Example of expressions
10474
10475 >>> mynumber = db.define_table('mynumber', Field('x', 'integer'))
10476 >>> db(mynumber).delete()
10477 0
10478 >>> for i in range(10): tmp = mynumber.insert(x=i)
10479 >>> db(mynumber).select(mynumber.x.sum())[0](mynumber.x.sum())
10480 45
10481
10482 >>> db(mynumber.x+2==5).select(mynumber.x + 2)[0](mynumber.x + 2)
10483 5
10484
10485 Output in csv
10486
10487 >>> print str(authored_papers.select(db.author.name, db.paper.title)).strip()
10488 author.name,paper.title\r
10489 Massimo,QCD
10490
10491 Delete all leftover tables
10492
10493 >>> DAL.distributed_transaction_commit(db)
10494
10495 >>> db.mynumber.drop()
10496 >>> db.authorship.drop()
10497 >>> db.author.drop()
10498 >>> db.paper.drop()
10499 """
10500
10501
10502
10503
10504 SQLField = Field
10505 SQLTable = Table
10506 SQLXorable = Expression
10507 SQLQuery = Query
10508 SQLSet = Set
10509 SQLRows = Rows
10510 SQLStorage = Row
10511 SQLDB = DAL
10512 GQLDB = DAL
10513 DAL.Field = Field
10514 DAL.Table = Table
10515
10516
10517
10518
10519
10520 -def geoPoint(x,y):
10521 return "POINT (%f %f)" % (x,y)
10522
10524 return "LINESTRING (%s)" % ','.join("%f %f" % item for item in line)
10525
10527 return "POLYGON ((%s))" % ','.join("%f %f" % item for item in line)
10528
10529
10530
10531
10532
10533 if __name__ == '__main__':
10534 import doctest
10535 doctest.testmod()
10536